Merge branch 0.9.4 into master. (#449)
Merge branch 0.9.4 into master. (#449)
diff --git a/.gitignore b/.gitignore
index 3eace27..fb88fe5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,6 +31,26 @@
resourceManager/resourcemanagerserver/target/
storage/storage/target/
+contextservice/cs-cache/target
+contextservice/cs-client/target
+contextservice/cs-common/target
+contextservice/cs-highavailable/target
+contextservice/cs-listener/target
+contextservice/cs-persistence/target
+contextservice/cs-search/target
+contextservice/cs-server/target
+contextservice/cs-ujes-client/target
+
+datasource/datasourcemanager/common/target
+datasource/datasourcemanager/server/target
+datasource/metadatamanager/common/target
+datasource/metadatamanager/server/target
+datasource/metadatamanager/service/elasticsearch/target
+datasource/metadatamanager/service/hive/target
+datasource/metadatamanager/service/mysql/target
+
+
+
ujes/client/target/
ujes/definedEngines/hive/engine/target/
ujes/definedEngines/hive/enginemanager/target/
diff --git a/assembly/pom.xml b/assembly/pom.xml
index b9170de..57c23a4 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,12 +21,11 @@
<parent>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>wedatasphere-linkis</artifactId>
- <packaging>pom</packaging>
<build>
<plugins>
<plugin>
diff --git a/assembly/public-module/pom.xml b/assembly/public-module/pom.xml
index ba3a041..a6b179e 100644
--- a/assembly/public-module/pom.xml
+++ b/assembly/public-module/pom.xml
@@ -1,11 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -53,6 +66,10 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/assembly/src/main/assembly/assembly.xml b/assembly/src/main/assembly/assembly.xml
index 9c8a032..ec4b928 100644
--- a/assembly/src/main/assembly/assembly.xml
+++ b/assembly/src/main/assembly/assembly.xml
@@ -268,8 +268,42 @@
<include>*.zip</include>
</includes>
</fileSet>
+ <!-- context service-->
+ <fileSet>
+ <directory>
+ ${project.parent.basedir}/contextservice/cs-server/target/
+ </directory>
+ <outputDirectory>share/linkis/linkis-cs-server/</outputDirectory>
+ <includes>
+ <include>*.zip</include>
+ </includes>
+ </fileSet>
+ <!--datasource manage server-->
+ <fileSet>
+ <directory>
+ ${project.parent.basedir}/datasource/datasourcemanager/server/target
+ </directory>
+ <outputDirectory>share/linkis/datasource/linkis-dsm-server/</outputDirectory>
+ <includes>
+ <include>*.zip</include>
+ </includes>
+ </fileSet>
+
+
+ <!--metadata manage server-->
+ <fileSet>
+ <directory>
+ ${project.parent.basedir}/datasource/metadatamanager/server/target
+ </directory>
+ <outputDirectory>share/linkis/datasource/linkis-mdm-server/</outputDirectory>
+ <includes>
+ <include>*.zip</include>
+ </includes>
+ </fileSet>
</fileSets>
+
+
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
diff --git a/bin/checkServices.sh b/bin/checkServices.sh
index 561277b..7e53bb5 100644
--- a/bin/checkServices.sh
+++ b/bin/checkServices.sh
@@ -29,9 +29,11 @@
MICRO_SERVICE_IP=$2
MICRO_SERVICE_PORT=$3
+
+source ${workDir}/bin/common.sh
local_host="`hostname --fqdn`"
-ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}')
+ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1')
function isLocal(){
if [ "$1" == "127.0.0.1" ];then
@@ -58,6 +60,7 @@
}
+
echo "Start to Check if your microservice:$MICRO_SERVICE_NAME is normal via telnet"
echo "--------------------------------------------------------------------------------------------------------------------------"
echo $MICRO_SERVICE_NAME
diff --git a/bin/common.sh b/bin/common.sh
new file mode 100644
index 0000000..c0ba755
--- /dev/null
+++ b/bin/common.sh
@@ -0,0 +1,74 @@
+#!/bin/sh
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#Actively load user env
+source ~/.bash_profile
+
+local_host="`hostname --fqdn`"
+
+ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}' | awk 'NR==1')
+
+function isLocal(){
+ if [ "$1" == "127.0.0.1" ];then
+ return 0
+ elif [ "$1" == "" ]; then
+ return 0
+ elif [ "$1" == "localhost" ]; then
+ return 0
+ elif [ "$1" == $local_host ]; then
+ return 0
+ elif [ "$1" == $ipaddr ]; then
+ return 0
+ fi
+ return 1
+}
+
+function executeCMD(){
+ isLocal $1
+ flag=$?
+ if [ $flag == "0" ];then
+ echo "Is local execution:$2"
+ eval $2
+ else
+ echo "Is remote execution:$2"
+ ssh -p $SSH_PORT $1 $2
+ fi
+
+}
+function copyFile(){
+ isLocal $1
+ flag=$?
+ src=$2
+ dest=$3
+ if [ $flag == "0" ];then
+ echo "Is local cp "
+ cp -r "$src" "$dest"
+ else
+ echo "Is remote cp "
+ scp -r -P $SSH_PORT "$src" $1:"$dest"
+ fi
+
+}
+
+function isSuccess(){
+if [ $? -ne 0 ]; then
+ echo "Failed to " + $1
+ exit 1
+else
+ echo "Succeed to" + $1
+fi
+}
\ No newline at end of file
diff --git a/bin/install.sh b/bin/install.sh
index 674163f..b646689 100644
--- a/bin/install.sh
+++ b/bin/install.sh
@@ -45,14 +45,7 @@
exit 1
fi
-function isSuccess(){
-if [ $? -ne 0 ]; then
- echo "Failed to " + $1
- exit 1
-else
- echo "Succeed to" + $1
-fi
-}
+source ${workDir}/bin/common.sh
function checkPythonAndJava(){
python --version
@@ -128,52 +121,9 @@
source ${DISTRIBUTION}
isSuccess "load config"
-local_host="`hostname --fqdn`"
-ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}')
-
-function isLocal(){
- if [ "$1" == "127.0.0.1" ];then
- return 0
- elif [ $1 == "localhost" ]; then
- return 0
- elif [ $1 == $local_host ]; then
- return 0
- elif [ $1 == $ipaddr ]; then
- return 0
- fi
- return 1
-}
-
-function executeCMD(){
- isLocal $1
- flag=$?
- if [ $flag == "0" ];then
- echo "Is local execution:$2"
- eval $2
- else
- echo "Is remote execution:$2"
- ssh -p $SSH_PORT $1 $2
- fi
-
-}
-function copyFile(){
- isLocal $1
- flag=$?
- src=$2
- dest=$3
- if [ $flag == "0" ];then
- echo "Is local cp "
- eval "cp -r $src $dest"
- else
- echo "Is remote cp "
- scp -r -P $SSH_PORT $src $1:$dest
- fi
-
-}
##install mode choice
-
if [ "$INSTALL_MODE" == "" ];then
echo "Please enter the mode selection such as: 1"
echo " 1: Lite"
@@ -185,7 +135,7 @@
fi
if [[ '1' = "$INSTALL_MODE" ]];then
- echo "You chose Lite installation mode"
+ echo "You chose Lite installation mode"
checkPythonAndJava
elif [[ '2' = "$INSTALL_MODE" ]];then
echo "You chose Simple installation mode"
@@ -233,7 +183,6 @@
elif [[ $WORKSPACE_USER_ROOT_PATH == hdfs://* ]];then
localRootDir=${WORKSPACE_USER_ROOT_PATH#hdfs://}
hdfs dfs -mkdir -p $localRootDir/$deployUser
- hdfs dfs -chmod -R 775 $localRootDir/$deployUser
else
echo "does not support $WORKSPACE_USER_ROOT_PATH filesystem types"
fi
@@ -251,7 +200,6 @@
elif [[ $HDFS_USER_ROOT_PATH == hdfs://* ]];then
localRootDir=${HDFS_USER_ROOT_PATH#hdfs://}
hdfs dfs -mkdir -p $localRootDir/$deployUser
- hdfs dfs -chmod -R 775 $localRootDir/$deployUser
else
echo "does not support $HDFS_USER_ROOT_PATH filesystem types"
fi
@@ -269,9 +217,8 @@
elif [[ $RESULT_SET_ROOT_PATH == hdfs://* ]];then
localRootDir=${RESULT_SET_ROOT_PATH#hdfs://}
hdfs dfs -mkdir -p $localRootDir/$deployUser
- hdfs dfs -chmod -R 775 $localRootDir/$deployUser
else
- echo "does not support $RESULT_SET_ROOT_PATH filesystem types"
+ echo "does not support $RESULT_SET_ROOT_PATH filesystem types"
fi
fi
isSuccess "create $RESULT_SET_ROOT_PATH directory"
@@ -335,20 +282,19 @@
isSuccess "create the dir of $SERVER_NAME"
fi
+if ! executeCMD $SERVER_IP "test -e $SERVER_HOME/module"; then
+ copyFile $SERVER_IP ${workDir}/share/linkis/module/module.zip $SERVER_HOME
+ isSuccess "cp module.zip"
+ executeCMD $SERVER_IP "cd $SERVER_HOME/;unzip -o module.zip > /dev/null;cd -"
+ isSuccess "unzip module.zip"
+fi
+
echo "$SERVER_NAME-step2:copy install package"
copyFile $SERVER_IP ${workDir}/share/$PACKAGE_DIR/$SERVER_NAME.zip $SERVER_HOME
isSuccess "copy ${SERVER_NAME}.zip"
executeCMD $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVER_NAME-bak; mv -f $SERVER_NAME $SERVER_NAME-bak;cd -"
executeCMD $SERVER_IP "cd $SERVER_HOME/;unzip -o $SERVER_NAME.zip > /dev/null; cd -"
isSuccess "unzip ${SERVER_NAME}.zip"
-if [ "$SERVER_NAME" != "linkis-gateway" ]
-then
- copyFile $SERVER_IP ${workDir}/share/linkis/module/module.zip $SERVER_HOME
- isSuccess "cp module.zip"
- executeCMD $SERVER_IP "cd $SERVER_HOME/;rm -rf modulebak;mv -f module modulebak;cd -"
- executeCMD $SERVER_IP "cd $SERVER_HOME/;unzip -o module.zip > /dev/null;cp -f module/lib/* $SERVER_HOME/$SERVER_NAME/lib/;cd -"
- isSuccess "unzip module.zip"
-fi
echo "$SERVER_NAME-step3:subsitution conf"
SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/application.yml
executeCMD $SERVER_IP "sed -i ${txt} \"s#port:.*#port: $SERVER_PORT#g\" $SERVER_CONF_PATH"
@@ -358,12 +304,32 @@
}
##function end
+##cp module to em lib
+function emExtraInstallModule(){
+ executeCMD $SERVER_IP "cd $SERVER_HOME/;cp -f module/lib/* $SERVER_HOME/$SERVER_NAME/lib/;cd -"
+ isSuccess "copy module"
+}
+##replace conf 1. replace if it exists 2.not exists add
+function replaceConf(){
+ option=$1
+ value=$2
+ file=$3
+ executeCMD $SERVER_IP "grep -q '^$option' $file && sed -i ${txt} 's/^$option.*/$option=$value/' $file || echo '$option=$value' >> $file"
+ isSuccess "copy module"
+}
+
+
##GateWay Install
PACKAGE_DIR=springcloud/gateway
SERVER_NAME=linkis-gateway
SERVER_IP=$GATEWAY_INSTALL_IP
SERVER_PORT=$GATEWAY_PORT
SERVER_HOME=$LINKIS_INSTALL_HOME
+if test -z "$SERVER_IP"
+then
+ GATEWAY_INSTALL_IP=$local_host
+fi
+
###install dir
installPackage
###update linkis.properties
@@ -393,8 +359,7 @@
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.workspace.filesystem.localuserrootpath.*#wds.linkis.workspace.filesystem.localuserrootpath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix.*#wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix=$HDFS_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_CONF_PATH"
-executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH"
-executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
##publicservice end
@@ -428,12 +393,14 @@
SERVER_HOME=$LINKIS_INSTALL_HOME
###install dir
installPackage
+emExtraInstallModule
###update linkis.properties
echo "$SERVER_NAME-step4:update linkis conf"
SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.enginemanager.sudo.script.*#wds.linkis.enginemanager.sudo.script=$SERVER_HOME/$SERVER_NAME/bin/rootScript.sh#g\" $SERVER_CONF_PATH"
SERVER_ENGINE_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis-engine.properties
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_ENGINE_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_ENGINE_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
@@ -450,6 +417,7 @@
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
##PythonEntrance install end
@@ -510,6 +478,24 @@
echo "<----------------$SERVER_NAME:end------------------->"
##metadata end
+##linkis-cs-server install
+PACKAGE_DIR=linkis/linkis-cs-server
+SERVER_NAME=linkis-cs-server
+SERVER_IP=$CS_INSTALL_IP
+SERVER_PORT=$CS_PORT
+SERVER_HOME=$LINKIS_INSTALL_HOME
+###install dir
+installPackage
+###update linkis.properties
+echo "$SERVER_NAME-step4:update linkis conf"
+SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
+executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH"
+executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH"
+executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH"
+isSuccess "subsitution linkis.properties of $SERVER_NAME"
+echo "<----------------$SERVER_NAME:end------------------->"
+##cs end
+
##HiveEM install
PACKAGE_DIR=linkis/ujes/hive
SERVER_NAME=linkis-ujes-hive-enginemanager
@@ -518,6 +504,7 @@
SERVER_HOME=$LINKIS_INSTALL_HOME
###install dir
installPackage
+emExtraInstallModule
###update linkis.properties
echo "$SERVER_NAME-step4:update linkis conf"
SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
@@ -527,7 +514,9 @@
SERVER_ENGINE_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis-engine.properties
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_ENGINE_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hive.config.dir.*#hive.config.dir=$HIVE_CONF_DIR#g\" $SERVER_ENGINE_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_ENGINE_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
+executeCMD $SERVER_IP "rm $SERVER_HOME/$SERVER_NAME/lib/guava-25.1-jre.jar"
executeCMD $SERVER_IP "rm $SERVER_HOME/$SERVER_NAME/lib/servlet-api-2.5.jar"
echo "<----------------$SERVER_NAME:end------------------->"
##HiveEM install end
@@ -544,6 +533,7 @@
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
##HiveEntrance install end
@@ -566,6 +556,7 @@
SERVER_HOME=$LINKIS_INSTALL_HOME
###install dir
installPackage
+emExtraInstallModule
###update linkis.properties
echo "$SERVER_NAME-step4:update linkis conf"
SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
@@ -576,6 +567,7 @@
SERVER_ENGINE_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis-engine.properties
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_ENGINE_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#spark.config.dir.*#spark.config.dir=$SPARK_CONF_DIR#g\" $SERVER_ENGINE_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_ENGINE_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
##SparkEM install end
@@ -592,6 +584,7 @@
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$HDFS_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
##SparkEntrance install end
@@ -600,6 +593,7 @@
##JDBCEntrance install
PACKAGE_DIR=linkis/ujes/jdbc
SERVER_NAME=linkis-ujes-jdbc-entrance
+SERVER_IP=$JDBC_INSTALL_IP
SERVER_PORT=$JDBC_ENTRANCE_PORT
###install dir
installPackage
@@ -609,6 +603,7 @@
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$HDFS_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
##JDBCEntrance install end
@@ -622,12 +617,14 @@
SERVER_HOME=$LINKIS_INSTALL_HOME
###install dir
installPackage
+emExtraInstallModule
###update linkis.properties
echo "$SERVER_NAME-step4:update linkis conf"
SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.enginemanager.sudo.script.*#wds.linkis.enginemanager.sudo.script=$SERVER_HOME/$SERVER_NAME/bin/rootScript.sh#g\" $SERVER_CONF_PATH"
SERVER_ENGINE_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis-engine.properties
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_ENGINE_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_ENGINE_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
@@ -641,8 +638,43 @@
echo "$SERVER_NAME-step4:update linkis conf"
SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
executeCMD $SERVER_IP "sed -i ${txt} \"s#\#hadoop.config.dir.*#hadoop.config.dir=$HADOOP_CONF_DIR#g\" $SERVER_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
isSuccess "subsitution linkis.properties of $SERVER_NAME"
echo "<----------------$SERVER_NAME:end------------------->"
##SHELLEntrance install end
+##Datasource Manager Server install
+PACKAGE_DIR=linkis/datasource/linkis-dsm-server
+SERVER_NAME=linkis-dsm-server
+SERVER_PORT=$DSM_PORT
+###install dir
+installPackage
+###update linkis.properties
+echo "$SERVER_NAME-step4:update linkis conf"
+SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
+executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH"
+executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH"
+executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH"
+executeCMD $SERVER_IP "sed -i ${txt} \"s#wds.linkis.server.dsm.admin.users.*#wds.linkis.server.dsm.admin.users=$deployUser#g\" $SERVER_CONF_PATH"
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
+isSuccess "subsitution linkis.properties of $SERVER_NAME"
+echo "<----------------$SERVER_NAME:end------------------->"
+##Datasource Manager Server install end
+
+
+
+##Metadata Manager Server install
+PACKAGE_DIR=linkis/datasource/linkis-mdm-server
+SERVER_NAME=linkis-mdm-server
+SERVER_PORT=$MDM_PORT
+###install dir
+installPackage
+###update linkis.properties
+echo "$SERVER_NAME-step4:update linkis conf"
+SERVER_CONF_PATH=$SERVER_HOME/$SERVER_NAME/conf/linkis.properties
+replaceConf "wds.linkis.gateway.url" "http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT" "$SERVER_CONF_PATH"
+isSuccess "subsitution linkis.properties of $SERVER_NAME"
+echo "<----------------$SERVER_NAME:end------------------->"
+##Metadata Manager Server install end
+
diff --git a/bin/start-all.sh b/bin/start-all.sh
index 2dc7884..05f5046 100644
--- a/bin/start-all.sh
+++ b/bin/start-all.sh
@@ -33,19 +33,13 @@
export DISTRIBUTION=${DISTRIBUTION:-"${CONF_DIR}/config.sh"}
#source $LINKIS_DSS_CONF_FILE
source ${DISTRIBUTION}
-function isSuccess(){
-if [ $? -ne 0 ]; then
- echo "Failed to " + $1
- exit 1
-else
- echo "Succeed to" + $1
-fi
-}
+
+source ${workDir}/bin/common.sh
local_host="`hostname --fqdn`"
-ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}')
+ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1')
function isLocal(){
if [ "$1" == "127.0.0.1" ];then
@@ -72,6 +66,7 @@
}
+
#if there is no LINKIS_INSTALL_HOME，we need to source config again
if [ -z ${LINKIS_INSTALL_HOME} ];then
echo "Info: LINKIS_INSTALL_HOME does not exist, we will source config"
@@ -88,6 +83,7 @@
echo "<-------------------------------->"
echo "Begin to start $SERVER_NAME"
SERVER_PATH=${APP_PREFIX}${SERVER_NAME}
+
SERVER_BIN=${LINKIS_INSTALL_HOME}/${SERVER_PATH}/bin
SERVER_LOCAL_START_CMD="dos2unix ${SERVER_BIN}/* > /dev/null 2>&1; dos2unix ${SERVER_BIN}/../conf/* > /dev/null 2>&1; sh ${SERVER_BIN}/start-${SERVER_NAME}.sh"
SERVER_REMOTE_START_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${SERVER_NAME}.sh > /dev/null 2>&1"
@@ -144,6 +140,21 @@
SERVER_IP=$BML_INSTALL_IP
startApp
+#cs-server
+SERVER_NAME="cs-server"
+SERVER_IP=$CS_INSTALL_IP
+startApp
+
+#datasource management
+SERVER_NAME="dsm-server"
+SERVER_IP=$DSM_INSTALL_IP
+startApp
+
+#metadata management
+SERVER_NAME="mdm-server"
+SERVER_IP=$MDM_INSTALL_IP
+startApp
+
#resourcemanager
SERVER_NAME="resourcemanager"
SERVER_IP=$RESOURCEMANAGER_INSTALL_IP
@@ -201,6 +212,7 @@
startApp
+
echo "start-all shell script executed completely"
echo "Start to check all dss microservice"
@@ -223,7 +235,7 @@
sh $workDir/bin/checkServices.sh $SERVER_NAME $SERVER_IP $SERVER_PORT
isSuccess "start $SERVER_NAME "
echo "<-------------------------------->"
-sleep 3
+sleep 5
}
SERVER_NAME="eureka"
SERVER_IP=$EUREKA_INSTALL_IP
@@ -257,6 +269,11 @@
SERVER_PORT=$BML_PORT
checkServer
+#cs-server
+SERVER_NAME="cs-server"
+SERVER_IP=$CS_INSTALL_IP
+checkServer
+
APP_PREFIX="linkis-ujes-"
SERVER_NAME=$APP_PREFIX"python-entrance"
SERVER_IP=$PYTHON_INSTALL_IP
@@ -293,4 +310,5 @@
SERVER_PORT=$JDBC_ENTRANCE_PORT
checkServer
+
echo "Linkis started successfully"
diff --git a/bin/stop-all.sh b/bin/stop-all.sh
index c04a633..098a577 100644
--- a/bin/stop-all.sh
+++ b/bin/stop-all.sh
@@ -45,7 +45,7 @@
local_host="`hostname --fqdn`"
-ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}')
+ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1')
function isLocal(){
if [ "$1" == "127.0.0.1" ];then
@@ -60,17 +60,9 @@
return 1
}
-function executeCMD(){
- isLocal $1
- flag=$?
- echo "Is local "$flag
- if [ $flag == "0" ];then
- eval $2
- else
- ssh -p $SSH_PORT $1 $2
- fi
-}
+source ${workDir}/bin/common.sh
+
#if there is no LINKIS_INSTALL_HOME，we need to source config again
if [ -z ${LINKIS_INSTALL_HOME} ];then
@@ -143,6 +135,20 @@
SERVER_IP=$METADATA_INSTALL_IP
stopApp
+#cs-server
+SERVER_NAME="cs-server"
+SERVER_IP=$CS_INSTALL_IP
+stopApp
+
+#datasource management
+SERVER_NAME="dsm-server"
+SERVER_IP=$DSM_INSTALL_IP
+stopApp
+
+#metadata management
+SERVER_NAME="mdm-server"
+SERVER_IP=$MDM_INSTALL_IP
+stopApp
APP_PREFIX="linkis-ujes-"
@@ -187,6 +193,10 @@
SERVER_IP=$HIVE_INSTALL_IP
stopApp
+#cs-server
+SERVER_NAME="cs-server"
+SERVER_IP=$CS_INSTALL_IP
+stopApp
#JDBCEntrance
@@ -194,6 +204,18 @@
SERVER_IP=$JDBC_INSTALL_IP
stopApp
+SERVER_NAME="pipeline-entrance"
+SERVER_IP=$PIPELINE_INSTALL_IP
+stopApp
+
+SERVER_NAME="pipeline-enginemanager"
+SERVER_IP=$PIPELINE_INSTALL_IP
+stopApp
+
+SERVER_NAME="io-enginemanager"
+SERVER_IP=$IO_INSTALL_IP
+stopApp
+
APP_PREFIX="linkis-"
diff --git a/bml/bml-engine-hook/pom.xml b/bml/bml-engine-hook/pom.xml
index 9b525ff..821a0a4 100644
--- a/bml/bml-engine-hook/pom.xml
+++ b/bml/bml-engine-hook/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -14,10 +14,10 @@
<dependencies>
<dependency>
- <groupId>com.webank.wedatasphere.linkis</groupId>
- <artifactId>linkis-common</artifactId>
- <scope>provided</scope>
- </dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis-bmlclient</artifactId>
diff --git a/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala b/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala
index 28f3cc0..22ed4b4 100644
--- a/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala
+++ b/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala
@@ -41,14 +41,12 @@
val pathType:String = "file://"
- override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest): Unit = {
- //1.删除工作目录以前的资源文件
- //2.下载资源到当前进程的工作目录
-
+ override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String = {
val workDir = BmlHookUtils.getCurrentWorkDir
val jobId = engineExecutorContext.getJobId
executeRequest match {
case resourceExecuteRequest:ResourceExecuteRequest => val resources = resourceExecuteRequest.resources
+ if (null == resources) return code
resources foreach {
case resource:util.Map[String, Object] => val fileName = resource.get(FILE_NAME_STR).toString
val resourceId = resource.get(RESOURCE_ID_STR).toString
@@ -74,5 +72,6 @@
}
case _ =>
}
+ if (StringUtils.isNotBlank(code)) code else executeRequest.code
}
}
diff --git a/bml/bmlclient/pom.xml b/bml/bmlclient/pom.xml
index cb188a6..5ea5c5c 100644
--- a/bml/bmlclient/pom.xml
+++ b/bml/bmlclient/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java
index 48ccd27..5b9620f 100644
--- a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java
+++ b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java
@@ -23,7 +23,7 @@
*/
public abstract class AbstractBmlClient implements BmlClient{
protected String user;
- protected java.util.Map<String, Object> properties;
+ protected Map<String, Object> properties;
public String getUser() {
return user;
diff --git a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java
index 1445926..e8d5a55 100644
--- a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java
+++ b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java
@@ -91,4 +91,11 @@
public BmlResourceVersionsResponse getVersions(String user, String resourceId);
+ /**
+ *
+ */
+ public BmlDeleteResponse deleteResource(String user, String resourceId, String version);
+
+ public BmlDeleteResponse deleteResource(String user, String resourceId);
+
}
diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala
index 76d3dc7..e7d6367 100644
--- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala
+++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala
@@ -290,6 +290,31 @@
}
+ /**
+ *
+ */
+ override def deleteResource(user: String, resourceId: String, version: String): BmlDeleteResponse = {
+ null
+ }
+
+ override def deleteResource(user: String, resourceId: String): BmlDeleteResponse = {
+ val deleteAction = BmlDeleteAction(resourceId)
+ deleteAction.getParameters.put("resourceId", resourceId)
+ val result = dwsClient.execute(deleteAction)
+ result match {
+ case bmlDeleteResult: BmlDeleteResult => val isSuccess= if (bmlDeleteResult.getStatus == 0) true else false
+ if (isSuccess){
+ BmlDeleteResponse(isSuccess)
+ }else{
+ logger.error(s"user $user update resource failed, status code is ${bmlDeleteResult.getStatusCode}")
+ BmlDeleteResponse(isSuccess)
+ }
+ case r:BmlResult => logger.error(s"result type ${r.getResultType} not match BmlResourceDownloadResult")
+ throw POSTResultNotMatchException()
+ case _ => throw POSTResultNotMatchException()
+ }
+ }
+
//todo 现在是为了通过编译
private def getInputStream(str: String):InputStream = {
null
diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala
index 2bb0ce5..cc2e6d9 100644
--- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala
+++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala
@@ -22,11 +22,7 @@
* Description:
*/
object BmlConfiguration {
- val GATEWAY_IP:CommonVars[String] =
- CommonVars[String]("wds.linkis.gateway.ip", "ip", "DWS gateway的ip地址")
- val GATEWAY_PORT:CommonVars[Int] =
- CommonVars[Int]("wds.linkis.gateway.port", 1000, "DWS gateway的端口")
val DWS_VERSION:CommonVars[String] = CommonVars[String]("wds.linkis.bml.dws.version", "v1")
@@ -47,6 +43,8 @@
val DOWNLOAD_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.download.url", "download")
+ val DELETE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.delete.url", "delete")
+
val GET_VERSIONS_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.getVersions.url", "getVersions")
val GET_BASIC_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.getBasic.url","getBasic")
diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala
index 5702970..4d335e3 100644
--- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala
+++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala
@@ -16,22 +16,22 @@
package com.webank.wedatasphere.linkis.bml.http
import com.webank.wedatasphere.linkis.bml.conf.BmlConfiguration
+import com.webank.wedatasphere.linkis.common.conf.Configuration
/**
* created by cooperyang on 2019/5/15
* Description:
*/
object HttpConf {
- val ip:String = BmlConfiguration.GATEWAY_IP.getValue
- val port:Int = BmlConfiguration.GATEWAY_PORT.getValue
- val schema:String = "http://"
- val gatewayInstance:String = schema + ip + ":" + port
- val urlPrefix:String = if (BmlConfiguration.URL_PREFIX.getValue.endsWith("/")) {
+
+ val gatewayInstance: String = Configuration.getGateWayURL()
+ val urlPrefix: String = if (BmlConfiguration.URL_PREFIX.getValue.endsWith("/")) {
BmlConfiguration.URL_PREFIX.getValue.substring(0, BmlConfiguration.URL_PREFIX.getValue.length - 1)
- }else BmlConfiguration.URL_PREFIX.getValue
+ } else BmlConfiguration.URL_PREFIX.getValue
val uploadURL:String = urlPrefix + "/" + BmlConfiguration.UPLOAD_URL.getValue
val downloadURL:String = urlPrefix + "/" + BmlConfiguration.DOWNLOAD_URL.getValue
+ val deleteURL:String = urlPrefix + "/" + BmlConfiguration.DELETE_URL
val updateVersionURL:String = urlPrefix + "/" + BmlConfiguration.UPDATE_VERSION_URL.getValue
val relateHdfsURL:String = gatewayInstance + urlPrefix + "/" + BmlConfiguration.RELATE_HDFS.getValue
val relateStorageURL:String = gatewayInstance + urlPrefix + "/" + BmlConfiguration.RELATE_STORAGE.getValue
diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala
index db40ec9..be915be 100644
--- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala
+++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala
@@ -19,11 +19,7 @@
import java.util
import com.webank.wedatasphere.linkis.bml.http.HttpConf
-
-import scala.collection.JavaConversions._
-import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.httpclient.request._
-import com.webank.wedatasphere.linkis.storage.FSFactory
/**
* created by cooperyang on 2019/5/23
@@ -173,5 +169,11 @@
+case class BmlDeleteAction(resourceId:String) extends BmlPOSTAction {
+ override def getRequestPayload: String = ""
+
+ override def getURL: String = HttpConf.deleteURL
+}
+
diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala
index b43ca48..a63943b 100644
--- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala
+++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala
@@ -21,8 +21,6 @@
import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult
import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult
-import scala.beans.BeanProperty
-
/**
* created by cooperyang on 2019/5/23
* Description:
@@ -106,3 +104,8 @@
def getDownloadedFileName:String = this.downloadedFileName
def setDownloadedFileName(downloadedFileName:String):Unit = this.downloadedFileName = downloadedFileName
}
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/bml/delete")
+class BmlDeleteResult extends BmlResult{
+
+}
\ No newline at end of file
diff --git a/bml/bmlcommon/pom.xml b/bml/bmlcommon/pom.xml
index 7aca4b2..7b0fa54 100644
--- a/bml/bmlcommon/pom.xml
+++ b/bml/bmlcommon/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala b/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala
index 25465ce..c8f29e4 100644
--- a/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala
+++ b/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala
@@ -74,3 +74,6 @@
case class BmlResourceVersionsResponse(isSuccess:Boolean,
resourceId:String,
resourceVersions: ResourceVersions) extends BmlResponse(isSuccess)
+
+
+case class BmlDeleteResponse(isSuccess:Boolean) extends BmlResponse(isSuccess)
\ No newline at end of file
diff --git a/bml/bmlserver/bin/start-bml.sh b/bml/bmlserver/bin/start-bml.sh
index 56cab6d..80cc775 100644
--- a/bml/bmlserver/bin/start-bml.sh
+++ b/bml/bmlserver/bin/start-bml.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/bml/bmlserver/pom.xml b/bml/bmlserver/pom.xml
index aff1dcc..ad028b1 100644
--- a/bml/bmlserver/pom.xml
+++ b/bml/bmlserver/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java
index 1b3a32e..35072a6 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java
@@ -20,7 +20,6 @@
import com.webank.wedatasphere.linkis.common.io.FsPath;
import com.webank.wedatasphere.linkis.storage.FSFactory;
import com.webank.wedatasphere.linkis.storage.utils.FileSystemUtils;
-
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java
index d28673e..eb2c867 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java
@@ -25,7 +25,7 @@
*/
public interface DownloadDao {
- void insertDownloadModel(@Param("downloadModel")DownloadModel downloadModel);
+ void insertDownloadModel(@Param("downloadModel") DownloadModel downloadModel);
}
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java
index be5e947..2c23ab2 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java
@@ -55,7 +55,7 @@
@Select("select start_byte from linkis_resources_version where resource_id = #{resourceId} and version = #{version}")
- long getStartByteForResource(@Param("resourceId") String resourceId, @Param("version")String version);
+ long getStartByteForResource(@Param("resourceId") String resourceId, @Param("version") String version);
@@ -75,14 +75,14 @@
@Select("select enable_flag from `linkis_resources_version` where resource_id = #{resourceId} and version = #{version}")
- int selectResourceVersionEnbleFlag(@Param("resourceId") String resourceId, @Param("version")String version);
+ int selectResourceVersionEnbleFlag(@Param("resourceId") String resourceId, @Param("version") String version);
/**
* 将resourceId对应的所有版本的enable_flag设为0，这样就不能继续访问该资源的任意版本
* @param resourceId resourceId
*/
@Update("update `linkis_resources_version` set enable_flag = 0 where resource_id = #{resourceId}")
- void deleteResource(@Param("resourceId")String resourceId);
+ void deleteResource(@Param("resourceId") String resourceId);
void batchDeleteResources(@Param("resourceIds") List<String> resourceIds);
@@ -92,6 +92,6 @@
- List<Version> selectVersionByPage(@Param("resourceId")String resourceId);
+ List<Version> selectVersionByPage(@Param("resourceId") String resourceId);
}
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java
index 072d377..412bfeb 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java
@@ -15,17 +15,9 @@
*/
package com.webank.wedatasphere.linkis.bml.restful;
-import com.webank.wedatasphere.linkis.bml.Entity.DownloadModel;
-import com.webank.wedatasphere.linkis.bml.Entity.Resource;
-import com.webank.wedatasphere.linkis.bml.Entity.ResourceTask;
-import com.webank.wedatasphere.linkis.bml.Entity.ResourceVersion;
-import com.webank.wedatasphere.linkis.bml.Entity.Version;
-import com.webank.wedatasphere.linkis.bml.common.Constant;
-import com.webank.wedatasphere.linkis.bml.service.BmlService;
-import com.webank.wedatasphere.linkis.bml.service.DownloadService;
-import com.webank.wedatasphere.linkis.bml.service.ResourceService;
-import com.webank.wedatasphere.linkis.bml.service.TaskService;
-import com.webank.wedatasphere.linkis.bml.service.VersionService;
+import com.webank.wedatasphere.linkis.bml.Entity.*;
+import com.webank.wedatasphere.linkis.bml.common.*;
+import com.webank.wedatasphere.linkis.bml.service.*;
import com.webank.wedatasphere.linkis.bml.threading.TaskState;
import com.webank.wedatasphere.linkis.bml.util.HttpRequestHelper;
import com.webank.wedatasphere.linkis.bml.vo.ResourceBasicVO;
@@ -33,8 +25,6 @@
import com.webank.wedatasphere.linkis.bml.vo.ResourceVersionsVO;
import com.webank.wedatasphere.linkis.common.exception.ErrorException;
import com.webank.wedatasphere.linkis.server.Message;
-import com.webank.wedatasphere.linkis.bml.common.*;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jackson.JsonNode;
@@ -45,25 +35,14 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.*;
/**
* created by cooperyang on 2019/5/14
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java
index 5aed763..cad6695 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java
@@ -53,7 +53,7 @@
* @param updateTime 操作时间
* @param errMsg 异常信息
*/
- void updateState2Failed( long taskId, String state, Date updateTime, String errMsg);
+ void updateState2Failed(long taskId, String state, Date updateTime, String errMsg);
ResourceTask createDeleteVersionTask(String resourceId, String version, String user, String ip);
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java
index 7b21870..332a8b5 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java
@@ -38,7 +38,7 @@
List<ResourceVersion> getAllResourcesViaSystem(String system, String user);
//分页查询VResourcesViaSystem
- public List<ResourceVersion> selectResourcesViaSystemByPage(int currentPage, int pageSize,String system, String user);
+ public List<ResourceVersion> selectResourcesViaSystemByPage(int currentPage, int pageSize, String system, String user);
void deleteResourceVersion(String resourceId, String version);
@@ -55,7 +55,7 @@
List<Version> getVersions(String resourceId);
//分页查询Version
- List<Version> selectVersionByPage(int currentPage, int pageSize,String resourceId);
+ List<Version> selectVersionByPage(int currentPage, int pageSize, String resourceId);
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java
index 06379f9..fe0a486 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java
@@ -24,7 +24,6 @@
import com.webank.wedatasphere.linkis.bml.dao.VersionDao;
import com.webank.wedatasphere.linkis.bml.service.ResourceService;
import com.webank.wedatasphere.linkis.common.exception.ErrorException;
-
import org.apache.commons.lang.StringUtils;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -35,13 +34,12 @@
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
+import javax.validation.constraints.NotNull;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import javax.validation.constraints.NotNull;
-
/**
* Created by cooperyang on 2019/5/17.
*/
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java
index cc7a3dc..830b6c0 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java
@@ -18,6 +18,7 @@
import com.webank.wedatasphere.linkis.bml.Entity.ResourceTask;
import com.webank.wedatasphere.linkis.bml.Entity.Version;
import com.webank.wedatasphere.linkis.bml.common.Constant;
+import com.webank.wedatasphere.linkis.bml.common.UpdateResourceException;
import com.webank.wedatasphere.linkis.bml.dao.ResourceDao;
import com.webank.wedatasphere.linkis.bml.dao.TaskDao;
import com.webank.wedatasphere.linkis.bml.dao.VersionDao;
@@ -25,7 +26,6 @@
import com.webank.wedatasphere.linkis.bml.service.TaskService;
import com.webank.wedatasphere.linkis.bml.service.VersionService;
import com.webank.wedatasphere.linkis.bml.threading.TaskState;
-import com.webank.wedatasphere.linkis.bml.common.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java
index 9d0142e..1003fa1 100644
--- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java
+++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java
@@ -28,7 +28,6 @@
import com.webank.wedatasphere.linkis.common.io.Fs;
import com.webank.wedatasphere.linkis.common.io.FsPath;
import com.webank.wedatasphere.linkis.storage.FSFactory;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
diff --git a/conf/config.sh b/conf/config.sh
index 79e7848..ef8a848 100644
--- a/conf/config.sh
+++ b/conf/config.sh
@@ -110,6 +110,19 @@
#BML_INSTALL_IP=127.0.0.1
BML_PORT=9113
+### cs
+#CS_INSTALL_IP=127.0.0.1
+CS_PORT=9116
+
+
+### datasource management server
+#DSM_INSTALL_IP=127.0.0.1
+DSM_PORT=9117
+
+### metadata management server
+#MDM_INSTALL_IP=127.0.0.1
+MDM_PORT=9118
+
########################################################################################
## LDAP is for enterprise authorization, if you just want to have a try, ignore it.
@@ -119,4 +132,4 @@
## java application default jvm memory
export SERVER_HEAP_SIZE="512M"
-LINKIS_VERSION=0.9.3
+LINKIS_VERSION=0.9.4
diff --git a/contextservice/cs-cache/pom.xml b/contextservice/cs-cache/pom.xml
new file mode 100644
index 0000000..ffbfac0
--- /dev/null
+++ b/contextservice/cs-cache/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-cache</artifactId>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-persistence</artifactId>
+ <version>${linkis.version}</version>
+
+ </dependency>
+
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.10</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>RELEASE</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-listener</artifactId>
+ <version>0.9.4</version>
+ <scope>compile</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/ContextCacheService.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/ContextCacheService.java
new file mode 100644
index 0000000..f8c655a
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/ContextCacheService.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 16:20
+ */
+public interface ContextCacheService {
+
+ ContextKeyValue put(ContextID contextID, ContextKeyValue csKeyValue) throws CSErrorException;
+
+ ContextKeyValue rest(ContextID contextID, ContextKey csKey);
+
+ ContextKeyValue get(ContextID contextID, ContextKey csKey);
+
+ List<ContextKeyValue> getValues(ContextID contextID, String keyword, ContextType csType);
+
+ List<ContextKeyValue> getAllLikes(ContextID contextID, String regex, ContextType csType);
+
+ List<ContextKeyValue> getAll(ContextID contextID);
+
+ List<ContextKeyValue> getAllByScope(ContextID contextID, ContextScope scope, ContextType csType);
+
+ List<ContextKeyValue> getAllByType(ContextID contextID, ContextType csType);
+
+ ContextKeyValue remove(ContextID contextID, ContextKey csKey);
+
+ void removeAll(ContextID contextID);
+
+ void removeAll(ContextID contextID, ContextScope scope, ContextType csType);
+
+ void removeAll(ContextID contextID, ContextType csType);
+
+ void removeByKeyPrefix(ContextID contextID, String preFix);
+
+ void removeByKeyPrefix(ContextID contextID, String preFix, ContextType csType);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/DefaultContextCacheService.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/DefaultContextCacheService.java
new file mode 100644
index 0000000..c964299
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/DefaultContextCacheService.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.DefaultContextCache;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.jvnet.hk2.annotations.Service;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 21:25
+ */
+@Component
+public class DefaultContextCacheService implements ContextCacheService {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultContextCacheService.class);
+
+
+ @Autowired
+ private ContextCache contextCache ;
+
+ @Override
+ public ContextKeyValue put(ContextID contextID, ContextKeyValue csKeyValue) throws CSErrorException {
+
+ if (null == contextID || null == csKeyValue) {
+ return null;
+ }
+
+ logger.info("Start to put contextKey({}) to ContextID({})", csKeyValue.getContextKey().getKey(), contextID.getContextId());
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ ContextKeyValue oldValue = contextIDValue.getContextKeyValueContext().put(csKeyValue);
+ logger.info("Finished to put contextKey({}) to ContextID({})", csKeyValue.getContextKey().getKey(), contextID.getContextId());
+
+ return oldValue;
+ }
+
+ @Override
+ public ContextKeyValue rest(ContextID contextID, ContextKey csKey) {
+ if (null == contextID || null == csKey) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ return contextIDValue.getContextKeyValueContext().remove(csKey);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to rest contextID(%s) of csKey(%s)",
+ contextID.getContextId(), csKey.getKey()), e);
+ }
+ return null;
+ }
+
+ @Override
+ public ContextKeyValue get(ContextID contextID, ContextKey csKey) {
+ if (null == contextID || null == csKey) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ return contextIDValue.getContextKeyValueContext().getContextKeyValue(csKey, csKey.getContextType());
+ } catch (Exception e) {
+ logger.error(String.format("Failed to get contextID(%s) of csKey(%s)",
+ contextID.getContextId(), csKey.getKey()), e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<ContextKeyValue> getValues(ContextID contextID, String keyword, ContextType csType) {
+ if (null == contextID || StringUtils.isBlank(keyword)) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ return contextIDValue.getContextKeyValueContext().getValues(keyword, csType);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to getValues contextID(%s) of keyword(%s)",
+ contextID.getContextId(), keyword), e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<ContextKeyValue> getAllLikes(ContextID contextID, String regex, ContextType csType) {
+ if (null == contextID || StringUtils.isBlank(regex)) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ return contextIDValue.getContextKeyValueContext().getAllLikes(regex, csType);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to getAllLikes contextID(%s) of regex(%s)",
+ contextID.getContextId(), regex), e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<ContextKeyValue> getAll(ContextID contextID) {
+
+ if (null == contextID) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ return contextIDValue.getContextKeyValueContext().getAll();
+ } catch (Exception e) {
+ logger.error(String.format("Failed to getAllByType contextID(%s)",
+ contextID.getContextId()), e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<ContextKeyValue> getAllByScope(ContextID contextID, ContextScope scope, ContextType csType) {
+ if (null == contextID && null == scope) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ List<ContextKeyValue> valueCSTypeList = contextIDValue.getContextKeyValueContext().getAllValues(csType);
+ if (CollectionUtils.isNotEmpty(valueCSTypeList)){
+ return valueCSTypeList.stream()
+ .filter(contextKeyValue -> scope.equals(contextKeyValue.getContextKey().getContextScope()))
+ .collect(Collectors.toList());
+ }
+ } catch (Exception e) {
+ logger.error(String.format("Failed to getAllByScope contextID(%s) of ContextScope(%s) of csType(%s)",
+ contextID.getContextId(), scope, csType), e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<ContextKeyValue> getAllByType(ContextID contextID, ContextType csType) {
+ if (null == contextID || StringUtils.isBlank(contextID.getContextId())) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ List<ContextKeyValue> allValues = contextIDValue.getContextKeyValueContext().getAllValues(csType);
+ return allValues;
+ } catch (Exception e) {
+ logger.error(String.format("Failed to getAllByType contextID(%s) of csType(%s)",
+ contextID.getContextId(), csType), e);
+ }
+ return null;
+ }
+
+ @Override
+ public ContextKeyValue remove(ContextID contextID, ContextKey csKey) {
+ if (null == contextID || csKey == null) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ return contextIDValue.getContextKeyValueContext().remove(csKey);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to remove contextID(%s) of csKey(%s)",
+ contextID.getContextId(), csKey.getKey()), e);
+ }
+ return null;
+ }
+
+ @Override
+ public void removeAll(ContextID contextID) {
+ if (null == contextID) {
+ return;
+ }
+ try {
+ contextCache.remove(contextID);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to removeAll contextID(%s)", contextID.getContextId()), e);
+ }
+ }
+
+ /**
+ * TODO This method has poor performance and is not implemented now
+ * @param contextID
+ * @param scope
+ * @param csType
+ */
+ @Override
+ public void removeAll(ContextID contextID, ContextScope scope, ContextType csType) {
+
+ }
+
+ @Override
+ public void removeAll(ContextID contextID, ContextType csType) {
+ if (null == contextID) {
+ return;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ contextIDValue.getContextKeyValueContext().removeAll(csType);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to removeAll contextID(%s) of csType(%s)", contextID.getContextId(), csType), e);
+ }
+ }
+
+ @Override
+ public void removeByKeyPrefix(ContextID contextID, String preFix) {
+ if (null == contextID || StringUtils.isBlank(preFix)) {
+ return ;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ contextIDValue.getContextKeyValueContext().removeByKeyPrefix(preFix);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to removeByKeyPrefix contextID(%s) of key preFix(%s)",
+ contextID.getContextId(), preFix), e);
+ }
+ }
+
+ @Override
+ public void removeByKeyPrefix(ContextID contextID, String preFix, ContextType csType) {
+ if (null == contextID || StringUtils.isBlank(preFix) || null == csType) {
+ return ;
+ }
+ try {
+ ContextIDValue contextIDValue = contextCache.getContextIDValue(contextID);
+ contextIDValue.getContextKeyValueContext().removeByKeyPrefix(preFix, csType);
+ } catch (Exception e) {
+ logger.error(String.format("Failed to removeByKeyPrefix contextID(%s) of key preFix(%s) and csyTye(%s)",
+ contextID.getContextId(), preFix, csType.name()), e);
+ }
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/ContextCache.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/ContextCache.java
new file mode 100644
index 0000000..c036626
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/ContextCache.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.ContextCacheMetric;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 16:45
+ */
+public interface ContextCache {
+
+ ContextIDValue getContextIDValue(ContextID contextID) throws CSErrorException;
+
+ void remove(ContextID contextID);
+
+ void put(ContextIDValue contextIDValue) throws CSErrorException;
+
+ Map<String, ContextIDValue> getAllPresent(List<ContextID> contextIDList);
+
+ void refreshAll() throws CSErrorException;
+
+ void putAll(List<ContextIDValue> contextIDValueList) throws CSErrorException;
+
+ ContextIDValue loadContextIDValue(ContextID contextID);
+
+ ContextCacheMetric getContextCacheMetric();
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/DefaultContextCache.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/DefaultContextCache.java
new file mode 100644
index 0000000..ab2eeb1
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/DefaultContextCache.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValueGenerator;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.ContextCacheMetric;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.ContextIDMetric;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.DefaultContextCacheMetric;
+import com.webank.wedatasphere.linkis.cs.listener.CSIDListener;
+import com.webank.wedatasphere.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus;
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType.*;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 17:50
+ */
+@Component
+public class DefaultContextCache implements ContextCache , CSIDListener {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultContextCache.class);
+
+ ContextAsyncListenerBus listenerBus = DefaultContextListenerManager.getInstance().getContextAsyncListenerBus();
+
+
+ @Autowired
+ private RemovalListener<String, ContextIDValue> contextIDRemoveListener;
+
+ @Autowired
+ private ContextIDValueGenerator contextIDValueGenerator ;
+
+ private Cache<String, ContextIDValue> cache = null;
+
+ private ContextCacheMetric contextCacheMetric = new DefaultContextCacheMetric();
+
+ @PostConstruct
+ private void init(){
+ this.cache = CacheBuilder.newBuilder().maximumSize(3000)
+ .removalListener(contextIDRemoveListener)
+ .recordStats().build();
+ }
+
+ @Override
+ public ContextIDValue getContextIDValue(ContextID contextID) throws CSErrorException {
+ if(null == contextID || StringUtils.isBlank(contextID.getContextId())) {
+ return null;
+ }
+ try {
+ ContextIDValue contextIDValue = cache.getIfPresent(contextID.getContextId());
+ if (contextIDValue == null){
+ contextIDValue = contextIDValueGenerator.createContextIDValue(contextID);
+ put(contextIDValue);
+ DefaultContextIDEvent defaultContextIDEvent = new DefaultContextIDEvent();
+ defaultContextIDEvent.setContextID(contextID);
+ defaultContextIDEvent.setOperateType(ADD);
+ listenerBus.post(defaultContextIDEvent);
+ }
+ DefaultContextIDEvent defaultContextIDEvent = new DefaultContextIDEvent();
+ defaultContextIDEvent.setContextID(contextID);
+ defaultContextIDEvent.setOperateType(ACCESS);
+ listenerBus.post(defaultContextIDEvent);
+ return contextIDValue;
+ } catch (Exception e){
+ String errorMsg = String.format("Failed to get contextIDValue of ContextID(%s)", contextID.getContextId());
+ logger.error(errorMsg);
+ throw new CSErrorException(97001, errorMsg, e);
+ }
+ }
+
+ @Override
+ public void remove(ContextID contextID) {
+ if ( null != contextID && StringUtils.isNotBlank(contextID.getContextId())){
+ logger.info("From cache to remove contextID:{}", contextID.getContextId());
+ cache.invalidate(contextID.getContextId());
+ }
+ }
+
+ @Override
+ public void put(ContextIDValue contextIDValue) throws CSErrorException {
+
+ if(contextIDValue != null && StringUtils.isNotBlank(contextIDValue.getContextID())){
+ cache.put(contextIDValue.getContextID(), contextIDValue);
+ }
+ }
+
+ @Override
+ public Map<String, ContextIDValue> getAllPresent(List<ContextID> contextIDList) {
+ List<String> contextIDKeys = contextIDList.stream().map(contextID -> {
+ if(StringUtils.isBlank(contextID.getContextId())) {
+ return null;
+ }
+ return contextID.getContextId();
+ }).filter(StringUtils :: isNotBlank).collect(Collectors.toList());
+ return cache.getAllPresent(contextIDKeys);
+ }
+
+ @Override
+ public void refreshAll() throws CSErrorException {
+ //TODO
+ }
+
+ @Override
+ public void putAll(List<ContextIDValue> contextIDValueList) throws CSErrorException {
+ for (ContextIDValue contextIDValue : contextIDValueList){
+ put(contextIDValue);
+ }
+ }
+
+ @Override
+ public ContextIDValue loadContextIDValue(ContextID contextID) {
+ return null;
+ }
+
+ @Override
+ public ContextCacheMetric getContextCacheMetric() {
+ return this.contextCacheMetric;
+ }
+
+ @Override
+ public void onCSIDAccess(ContextIDEvent contextIDEvent) {
+ ContextID contextID = contextIDEvent.getContextID();
+ try {
+ ContextIDValue contextIDValue = getContextIDValue(contextID);
+ ContextIDMetric contextIDMetric = contextIDValue.getContextIDMetric();
+
+ contextIDMetric.setLastAccessTime(System.currentTimeMillis());
+ contextIDMetric.addCount();
+ getContextCacheMetric().addCount();
+ } catch (CSErrorException e) {
+ logger.error("Failed to deal CSIDAccess event csid is {}", contextID.getContextId());
+ }
+ }
+
+ @Override
+ public void onCSIDADD(ContextIDEvent contextIDEvent) {
+ logger.info("deal contextID ADD event of {}", contextIDEvent.getContextID());
+ getContextCacheMetric().addCount();
+ getContextCacheMetric().setCachedCount(getContextCacheMetric().getCachedCount() + 1);
+ logger.info("Now, The cachedCount is (%d)", getContextCacheMetric().getCachedCount());
+ }
+
+ @Override
+ public void onCSIDRemoved(ContextIDEvent contextIDEvent) {
+ logger.info("deal contextID remove event of {}", contextIDEvent.getContextID());
+ getContextCacheMetric().setCachedCount(getContextCacheMetric().getCachedCount() - 1);
+ logger.info("Now, The cachedCount is (%d)", getContextCacheMetric().getCachedCount());
+ }
+
+ @Override
+ public void onEventError( Event event, Throwable t) {
+ logger.error("Failed to deal event", t);
+ }
+
+ @Override
+ public void onEvent(Event event) {
+ DefaultContextIDEvent defaultContextIDEvent = null;
+ if (event != null && event instanceof DefaultContextIDEvent){
+ defaultContextIDEvent = (DefaultContextIDEvent) event;
+ }
+ if (null == defaultContextIDEvent) {
+ return;
+ }
+ if (ADD.equals(defaultContextIDEvent.getOperateType())){
+ onCSIDADD(defaultContextIDEvent);
+ } else if (DELETE.equals(defaultContextIDEvent.getOperateType())) {
+ onCSIDRemoved(defaultContextIDEvent);
+ } else {
+ onCSIDAccess(defaultContextIDEvent);
+ }
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/ContextIDValue.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/ContextIDValue.java
new file mode 100644
index 0000000..f149e5b
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/ContextIDValue.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.csid;
+
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.ContextKeyValueContext;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.ContextIDMetric;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 16:59
+ */
+public interface ContextIDValue {
+
+ String getContextID();
+
+ ContextKeyValueContext getContextKeyValueContext();
+
+ void refresh();
+
+ ContextIDMetric getContextIDMetric();
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/ContextIDValueGenerator.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/ContextIDValueGenerator.java
new file mode 100644
index 0000000..ed50f7d
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/ContextIDValueGenerator.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.csid;
+
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 14:47
+ */
+public interface ContextIDValueGenerator {
+
+ ContextIDValue createContextIDValue(ContextID contextID) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/impl/ContextIDValueGeneratorImpl.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/impl/ContextIDValueGeneratorImpl.java
new file mode 100644
index 0000000..3ffcd8f
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/impl/ContextIDValueGeneratorImpl.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.impl;
+
+import com.webank.wedatasphere.linkis.common.exception.FatalException;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValueGenerator;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.ContextKeyValueContext;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextIDCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextKeyCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import com.webank.wedatasphere.linkis.cs.persistence.ContextPersistenceManager;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextIDListenerPersistence;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextKeyListenerPersistence;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextMapPersistence;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Lookup;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 14:52
+ */
+@Component
+public abstract class ContextIDValueGeneratorImpl implements ContextIDValueGenerator {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextIDValueGeneratorImpl.class);
+
+
+ @Lookup
+ protected abstract ContextKeyValueContext getContextKeyValueContext();
+
+ @Autowired
+ private ContextPersistenceManager contextPersistenceManager;
+
+
+ private ContextMapPersistence contextMapPersistence;
+
+ private ContextIDListenerPersistence contextIDListenerPersistence;
+
+ private ContextKeyListenerPersistence contextKeyListenerPersistence;
+
+ private DefaultContextIDCallbackEngine contextIDCallbackEngine;
+
+ private DefaultContextKeyCallbackEngine contextKeyCallbackEngine;
+
+ @PostConstruct
+ void init() throws FatalException{
+ try {
+ this.contextIDCallbackEngine = DefaultContextListenerManager.getInstance().getContextIDCallbackEngine();
+ this.contextKeyCallbackEngine = DefaultContextListenerManager.getInstance().getContextKeyCallbackEngine();
+ this.contextMapPersistence = contextPersistenceManager.getContextMapPersistence();
+ this.contextIDListenerPersistence = contextPersistenceManager.getContextIDListenerPersistence();
+ this.contextKeyListenerPersistence = contextPersistenceManager.getContextKeyListenerPersistence();
+ } catch (Exception e) {
+ throw new FatalException(97001, "Failed to get proxy of contextMapPersistence");
+ }
+ }
+
+
+ @Override
+ public ContextIDValue createContextIDValue(ContextID contextID) throws CSErrorException {
+ logger.info("Start to createContextIDValue of ContextID({}) ", contextID.getContextId());
+
+ if (contextMapPersistence == null ) {
+ throw new CSErrorException(97001, "Failed to get proxy of contextMapPersistence");
+ }
+
+ List<ContextKeyValue> contextKeyValueList = contextMapPersistence.getAll(contextID);
+
+ ContextKeyValueContext contextKeyValueContext = getContextKeyValueContext();
+ contextKeyValueContext.setContextID(contextID);
+ contextKeyValueContext.putAll(contextKeyValueList);
+
+
+ try {
+ logger.info("For contextID({}) register contextKeyListener", contextID.getContextId());
+ List<ContextKeyListenerDomain> contextKeyListenerPersistenceAll = this.contextKeyListenerPersistence.getAll(contextID);
+ if (CollectionUtils.isNotEmpty(contextKeyListenerPersistenceAll)){
+ for (ContextKeyListenerDomain contextKeyListenerDomain : contextKeyListenerPersistenceAll){
+ this.contextKeyCallbackEngine.registerClient(contextKeyListenerDomain);
+ }
+ }
+ logger.info("For contextID({}) register contextIDListener", contextID.getContextId());
+ List<ContextIDListenerDomain> contextIDListenerPersistenceAll = this.contextIDListenerPersistence.getAll(contextID);
+
+ if (CollectionUtils.isNotEmpty(contextIDListenerPersistenceAll)){
+ for (ContextIDListenerDomain contextIDListenerDomain : contextIDListenerPersistenceAll){
+ this.contextIDCallbackEngine.registerClient(contextIDListenerDomain);
+ }
+ }
+ } catch (Throwable e) {
+ logger.error("Failed to register listener: ", e);
+ }
+
+ logger.info("Finished to createContextIDValue of ContextID({}) ", contextID.getContextId());
+ return new ContextIDValueImpl(contextID.getContextId(), contextKeyValueContext);
+ }
+
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/impl/ContextIDValueImpl.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/impl/ContextIDValueImpl.java
new file mode 100644
index 0000000..7c271e6
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/csid/impl/ContextIDValueImpl.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.impl;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.ContextKeyValueContext;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.ContextIDMetric;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.DefaultContextIDMetric;
+import com.webank.wedatasphere.linkis.cs.contextcache.metric.SizeEstimator;
+import com.webank.wedatasphere.linkis.cs.listener.CSKeyListener;
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextKeyEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType.*;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 20:56
+ */
+public class ContextIDValueImpl implements ContextIDValue, CSKeyListener {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextIDValueImpl.class);
+
+
+ private String contextID;
+
+ private ContextKeyValueContext contextKeyValueContext;
+
+ private ContextIDMetric contextIDMetric = new DefaultContextIDMetric();
+
+ public ContextIDValueImpl() {
+
+ }
+
+ public ContextIDValueImpl(String contextID, ContextKeyValueContext contextKeyValueContext) {
+ this.contextID = contextID;
+ this.contextKeyValueContext = contextKeyValueContext;
+ }
+
+ @Override
+ public String getContextID() {
+ return this.contextID;
+ }
+
+ @Override
+ public ContextKeyValueContext getContextKeyValueContext() {
+ return this.contextKeyValueContext;
+ }
+
+ @Override
+ public void refresh() {
+ //TODO
+ }
+
+ @Override
+ public ContextIDMetric getContextIDMetric() {
+ return this.contextIDMetric;
+ }
+
+
+ @Override
+ public void onEvent(Event event) {
+ DefaultContextKeyEvent defaultContextKeyEvent = null;
+ if (event != null && event instanceof DefaultContextKeyEvent){
+ defaultContextKeyEvent = (DefaultContextKeyEvent) event;
+ }
+ if (null == defaultContextKeyEvent) {
+ return;
+ }
+ if (ACCESS.equals(defaultContextKeyEvent.getOperateType())){
+ onCSKeyAccess(defaultContextKeyEvent);
+ } else {
+ onCSKeyUpdate(defaultContextKeyEvent);
+ }
+ }
+
+ @Override
+ public void onCSKeyUpdate(ContextKeyEvent contextKeyEvent) {
+
+ DefaultContextKeyEvent defaultContextKeyEvent = (DefaultContextKeyEvent) contextKeyEvent;
+ logger.debug("Start to deal csKeyEvent of csID({})", this.contextID);
+ if (ADD == defaultContextKeyEvent.getOperateType()) {
+ Long size = SizeEstimator.estimate(defaultContextKeyEvent.getContextKeyValue());
+ this.contextIDMetric.setMemory(getContextIDMetric().getMemory() + size);
+ } else if (DELETE == defaultContextKeyEvent.getOperateType()) {
+ Long size = SizeEstimator.estimate(defaultContextKeyEvent.getContextKeyValue());
+ this.contextIDMetric.setMemory(getContextIDMetric().getMemory() - size);
+ } else if (REMOVEALL == defaultContextKeyEvent.getOperateType()) {
+ Long size = SizeEstimator.estimate(getContextKeyValueContext());
+ this.contextIDMetric.setMemory(size);
+ } else {
+ long size = SizeEstimator.estimate(defaultContextKeyEvent.getContextKeyValue()) - SizeEstimator.estimate(defaultContextKeyEvent.getOldValue());
+ this.contextIDMetric.setMemory(getContextIDMetric().getMemory() - size);
+ }
+ logger.info("Now, The Memory of ContextID({}) are %d", contextID, getContextIDMetric().getMemory());
+ logger.debug("Finished to deal csKeyEvent of csID({})", this.contextID);
+ }
+
+ @Override
+ public void onCSKeyAccess(ContextKeyEvent contextKeyEvent) {
+ //TODO null
+ }
+
+ @Override
+ public void onEventError( Event event, Throwable t) {
+ logger.error("Failed to deal event", t);
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/ContextKeyValueContext.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/ContextKeyValueContext.java
new file mode 100644
index 0000000..67180ab
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/ContextKeyValueContext.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSWarnException;
+import com.webank.wedatasphere.linkis.cs.contextcache.index.ContextInvertedIndexSet;
+import com.webank.wedatasphere.linkis.cs.contextcache.parser.ContextKeyValueParser;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 10:24
+ */
+public interface ContextKeyValueContext {
+
+ ContextID getContextID();
+
+ void setContextID(ContextID contextID) throws CSWarnException;
+
+ ContextInvertedIndexSet getContextInvertedIndexSet();
+
+ ContextValueMapSet getContextValueMapSet();
+
+ ContextKeyValueParser getContextKeyValueParser();
+
+ ContextKeyValue put(ContextKeyValue contextKeyValue);
+
+ ContextKeyValue getContextKeyValue(ContextKey contextKey, ContextType contextType);
+
+ List<ContextKeyValue> getValues(String keyword, ContextType contextType);
+
+ List<ContextKeyValue> getValues(List<String> contextKeys, ContextType contextType);
+
+ List<ContextKeyValue> getAllValues(ContextType contextType);
+
+ List<ContextKeyValue> getAllLikes(String regex,ContextType contextType);
+
+ List<ContextKeyValue> getAll();
+
+ ContextKeyValue remove(ContextKey contextKey);
+
+ Map<String, ContextKeyValue> removeAll(ContextType contextType);
+
+ Boolean putAll(List<ContextKeyValue> contextKeyValueList);
+
+ void removeByKeyPrefix(String preFix);
+
+ void removeByKeyPrefix(String preFix, ContextType csType);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/ContextValueMapSet.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/ContextValueMapSet.java
new file mode 100644
index 0000000..91e7512
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/ContextValueMapSet.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey;
+
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author peacewong
+ * @date 2020/2/11 15:20
+ */
+public interface ContextValueMapSet {
+
+ Map<String, ContextKeyValue> getContextValueMap(ContextType contextType);
+
+ ContextKeyValue put(ContextKeyValue contextKeyValue);
+
+ ContextKeyValue getByContextKey(ContextKey contextKey, ContextType contextType);
+
+ ContextKeyValue getByContextKey(String contextKey, ContextType contextType);
+
+ List<ContextKeyValue> getByContextKeys(List<String> contextKeys, ContextType contextType);
+
+ List<ContextKeyValue> getAllValuesByType(ContextType contextType);
+
+ List<ContextKeyValue> getAllLikes(String regex, ContextType contextType);
+
+ List<ContextKeyValue> getAll();
+
+ ContextKeyValue remove(String contextKey, ContextType contextType);
+
+ Map<String, ContextKeyValue> removeAll(ContextType contextType);
+
+ List<ContextKey> findByKeyPrefix(String preFix);
+
+ List<ContextKey> findByKeyPrefix(String preFix, ContextType csType);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/impl/ContextValueMapSetImpl.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/impl/ContextValueMapSetImpl.java
new file mode 100644
index 0000000..4c00aa2
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/impl/ContextValueMapSetImpl.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.ContextValueMapSet;
+import com.webank.wedatasphere.linkis.cs.contextcache.index.ContextInvertedIndexSetImpl;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * @author peacewong
+ * @date 2020/2/11 20:06
+ */
+public class ContextValueMapSetImpl implements ContextValueMapSet {
+
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextInvertedIndexSetImpl.class);
+
+ Map<String, Map<String, ContextKeyValue>> contextValueMapSet = new HashMap<>();
+
+ @Override
+ public Map<String, ContextKeyValue> getContextValueMap(ContextType contextType) {
+ if(contextType == null) {
+ contextType = ContextType.METADATA;
+ }
+ String csType = contextType.name();
+ if (!contextValueMapSet.containsKey(csType)) {
+ synchronized (csType.intern()) {
+ if (!contextValueMapSet.containsKey(csType)) {
+ logger.info("For ContextType({}) init ContextValueMap", csType);
+ contextValueMapSet.put(csType, new HashMap<String, ContextKeyValue>(16));
+ }
+ }
+ }
+ return contextValueMapSet.get(csType);
+ }
+
+ @Override
+ public ContextKeyValue put(ContextKeyValue contextKeyValue) {
+ if (contextKeyValue.getContextKey() != null
+ && contextKeyValue.getContextValue() != null
+ && StringUtils.isNotBlank(contextKeyValue.getContextKey().getKey())) {
+ return getContextValueMap(contextKeyValue.getContextKey().getContextType()).put(contextKeyValue.getContextKey().getKey(), contextKeyValue);
+ }
+ return null;
+ }
+
+
+ @Override
+ public ContextKeyValue getByContextKey(ContextKey contextKey, ContextType contextType) {
+ if (contextKey != null && StringUtils.isNotBlank(contextKey.getKey())) {
+ return getContextValueMap(contextType).get(contextKey.getKey());
+ }
+ return null;
+ }
+
+ @Override
+ public ContextKeyValue getByContextKey(String contextKey, ContextType contextType) {
+ if (StringUtils.isNotBlank(contextKey)) {
+ return getContextValueMap(contextType).get(contextKey);
+ }
+ return null;
+ }
+
+ @Override
+ public List<ContextKeyValue> getByContextKeys(List<String> contextKeys, ContextType contextType) {
+ if(CollectionUtils.isEmpty(contextKeys)) {
+ return null;
+ }
+ List<ContextKeyValue> contextKeyValueList = new ArrayList<>();
+ for (String contextKey : contextKeys) {
+ ContextKeyValue contextKeyValue = getByContextKey(contextKey, contextType);
+ if (null != contextKeyValue) {
+ contextKeyValueList.add(contextKeyValue);
+ }
+ }
+ return contextKeyValueList;
+ }
+
+ @Override
+ public List<ContextKeyValue> getAllValuesByType(ContextType contextType) {
+
+ Collection<ContextKeyValue> values = getContextValueMap(contextType).values();
+
+ if (null != values){
+ return new ArrayList<>(values);
+ }
+ return null;
+ }
+
+
+ @Override
+ public List<ContextKeyValue> getAllLikes(String regex, ContextType contextType) {
+ if (StringUtils.isBlank(regex)){
+ return null;
+ }
+ Map<String, ContextKeyValue> contextValueMap = getContextValueMap(contextType);
+ List<ContextKeyValue> contextKeyValueList = new ArrayList<>();
+ Iterator<String> iterator = contextValueMap.keySet().iterator();
+ while (iterator.hasNext()){
+ String key = iterator.next();
+ if(key.matches(regex)){
+ contextKeyValueList.add(contextValueMap.get(key));
+ }
+ }
+ return contextKeyValueList;
+ }
+
+ @Override
+ public List<ContextKeyValue> getAll() {
+ List<ContextKeyValue> contextKeyValueList = new ArrayList<>();
+
+ for (Map<String, ContextKeyValue> contextKeyValueMap : contextValueMapSet.values()){
+ contextKeyValueList.addAll(contextKeyValueMap.values());
+ }
+ return contextKeyValueList;
+ }
+
+
+ @Override
+ public ContextKeyValue remove(String contextKey, ContextType contextType) {
+ if (StringUtils.isNotBlank(contextKey)){
+ return getContextValueMap(contextType).remove(contextKey);
+ }
+ return null;
+ }
+
+ @Override
+ public Map<String, ContextKeyValue> removeAll(ContextType contextType) {
+ return contextValueMapSet.remove(contextType);
+ }
+
+ @Override
+ public List<ContextKey> findByKeyPrefix(String preFix) {
+ if (StringUtils.isBlank(preFix)){
+ return null;
+ }
+ List<ContextKey> contextKeyValueList = new ArrayList<>();
+ for (Map<String, ContextKeyValue> contextKeyValueMap : contextValueMapSet.values()) {
+ Iterator<String> iterator = contextKeyValueMap.keySet().iterator();
+ while (iterator.hasNext()) {
+ String key = iterator.next();
+ if (key.startsWith(preFix)) {
+ contextKeyValueList.add(contextKeyValueMap.get(key).getContextKey());
+ }
+ }
+ }
+ return contextKeyValueList;
+ }
+
+ @Override
+ public List<ContextKey> findByKeyPrefix(String preFix, ContextType csType) {
+ if (StringUtils.isBlank(preFix)){
+ return null;
+ }
+ Map<String, ContextKeyValue> contextValueMap = getContextValueMap(csType);
+ List<ContextKey> contextKeyValueList = new ArrayList<>();
+ Iterator<String> iterator = contextValueMap.keySet().iterator();
+ while (iterator.hasNext()){
+ String key = iterator.next();
+ if(key.startsWith(preFix)){
+ contextKeyValueList.add(contextValueMap.get(key).getContextKey());
+ }
+ }
+ return contextKeyValueList;
+ }
+
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/impl/DefaultContextKeyValueContext.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/impl/DefaultContextKeyValueContext.java
new file mode 100644
index 0000000..3dfce39
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/cskey/impl/DefaultContextKeyValueContext.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSWarnException;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.ContextKeyValueContext;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.cskey.ContextValueMapSet;
+import com.webank.wedatasphere.linkis.cs.contextcache.index.ContextInvertedIndexSet;
+import com.webank.wedatasphere.linkis.cs.contextcache.index.ContextInvertedIndexSetImpl;
+import com.webank.wedatasphere.linkis.cs.contextcache.parser.ContextKeyValueParser;
+import com.webank.wedatasphere.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus;
+import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 15:34
+ */
+@Component
+@Scope("prototype")
+public class DefaultContextKeyValueContext implements ContextKeyValueContext {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultContextKeyValueContext.class);
+
+ ContextInvertedIndexSet contextInvertedIndexSet = new ContextInvertedIndexSetImpl();
+
+ ContextValueMapSet contextValueMapSet = new ContextValueMapSetImpl();
+
+ ContextAsyncListenerBus listenerBus = DefaultContextListenerManager.getInstance().getContextAsyncListenerBus();
+
+ private ContextID contextID;
+
+ @Autowired
+ ContextKeyValueParser contextKeyValueParser ;
+
+ @Override
+ public ContextID getContextID() {
+ return this.contextID;
+ }
+
+ @Override
+ public void setContextID(ContextID contextID) throws CSWarnException {
+ if (this.contextID == null){
+ this.contextID = contextID;
+ } else {
+ logger.error("Do not set contextID repeatedly.The current context is {}", this.contextID.getContextId());
+ throw new CSWarnException(97001, "Do not set contextID repeatedly");
+ }
+
+ }
+
+ @Override
+ public ContextInvertedIndexSet getContextInvertedIndexSet() {
+ return this.contextInvertedIndexSet;
+ }
+
+ @Override
+ public ContextValueMapSet getContextValueMapSet() {
+ return this.contextValueMapSet;
+ }
+
+ @Override
+ public ContextKeyValueParser getContextKeyValueParser() {
+ return this.contextKeyValueParser;
+ }
+
+
+ @Override
+ public ContextKeyValue put(ContextKeyValue contextKeyValue) {
+ ContextKey contextKey = contextKeyValue.getContextKey();
+ if (contextKey == null || StringUtils.isBlank(contextKey.getKey())) {
+ return null;
+ }
+ ContextKeyValue oldValue = getContextValueMapSet().put(contextKeyValue);
+ Set<String> keyWords = getContextKeyValueParser().parse(contextKeyValue);
+
+ getContextInvertedIndexSet().addKeywords(keyWords, contextKey.getKey(), contextKey.getContextType());
+ DefaultContextKeyEvent defaultContextKeyEvent = new DefaultContextKeyEvent();
+ defaultContextKeyEvent.setContextID(contextID);
+ defaultContextKeyEvent.setContextKeyValue(contextKeyValue);
+ defaultContextKeyEvent.setOldValue(oldValue);
+ if (null != oldValue ){
+ defaultContextKeyEvent.setOperateType(OperateType.UPDATE);
+ } else {
+ defaultContextKeyEvent.setOperateType(OperateType.ADD);
+ }
+ listenerBus.post(defaultContextKeyEvent);
+ return oldValue;
+ }
+
+ @Override
+ public ContextKeyValue remove(ContextKey contextKey) {
+ if (contextKey == null || StringUtils.isBlank(contextKey.getKey())) {
+ return null;
+ }
+ ContextKeyValue contextKeyValue = getContextValueMapSet().remove(contextKey.getKey(), contextKey.getContextType());
+ if (null == contextKeyValue){
+ return null;
+ }
+ Set<String> keyWords = getContextKeyValueParser().parse(contextKeyValue);
+ Iterator<String> iterator = keyWords.iterator();
+ ContextInvertedIndexSet contextInvertedIndexSet = getContextInvertedIndexSet();
+ while (iterator.hasNext()) {
+ contextInvertedIndexSet.remove(iterator.next(), contextKey.getKey(), contextKey.getContextType());
+ }
+ logger.info("Succeed to remove contextKey of {}", contextKey.getKey());
+ DefaultContextKeyEvent defaultContextKeyEvent = new DefaultContextKeyEvent();
+ defaultContextKeyEvent.setContextID(contextID);
+ defaultContextKeyEvent.setContextKeyValue(contextKeyValue);
+ defaultContextKeyEvent.setOperateType(OperateType.DELETE);
+ listenerBus.post(defaultContextKeyEvent);
+ return contextKeyValue;
+ }
+
+ @Override
+ public ContextKeyValue getContextKeyValue(ContextKey contextKey, ContextType contextType) {
+ return getContextValueMapSet().getByContextKey(contextKey, contextType);
+ }
+
+
+ @Override
+ public List<ContextKeyValue> getValues(String keyword, ContextType contextType) {
+ List<String> contextKeys = getContextInvertedIndexSet().getContextKeys(keyword, contextType);
+ return getValues(contextKeys, contextType);
+ }
+
+ @Override
+ public List<ContextKeyValue> getValues(List<String> contextKeys, ContextType contextType) {
+ return getContextValueMapSet().getByContextKeys(contextKeys, contextType);
+ }
+
+ @Override
+ public List<ContextKeyValue> getAllValues(ContextType contextType) {
+ return getContextValueMapSet().getAllValuesByType(contextType);
+ }
+
+ @Override
+ public List<ContextKeyValue> getAllLikes(String regex, ContextType contextType) {
+ return getContextValueMapSet().getAllLikes(regex, contextType);
+ }
+
+ @Override
+ public List<ContextKeyValue> getAll() {
+
+ return contextValueMapSet.getAll();
+ }
+
+
+ @Override
+ public Map<String, ContextKeyValue> removeAll(ContextType contextType) {
+
+ DefaultContextKeyEvent defaultContextKeyEvent = new DefaultContextKeyEvent();
+ defaultContextKeyEvent.setContextID(contextID);
+ defaultContextKeyEvent.setOperateType(OperateType.REMOVEALL);
+ listenerBus.post(defaultContextKeyEvent);
+ getContextInvertedIndexSet().removeAll(contextType);
+ logger.warn("ContextID({}) removeAll contextKey of contextType({})", contextID.getContextId(), contextType.name());
+ return getContextValueMapSet().removeAll(contextType);
+ }
+
+
+
+ @Override
+ public Boolean putAll(List<ContextKeyValue> contextKeyValueList) {
+ for (ContextKeyValue contextKeyValue : contextKeyValueList) {
+ put(contextKeyValue);
+ }
+ return true;
+ }
+
+ @Override
+ public void removeByKeyPrefix(String preFix) {
+ List<ContextKey> removeKeys = getContextValueMapSet().findByKeyPrefix(preFix);
+ if (CollectionUtils.isNotEmpty(removeKeys)){
+ for (ContextKey key : removeKeys){
+ remove(key);
+ }
+ logger.warn("Remove keyValue by key preFix: " + preFix);
+ }
+
+ }
+
+ @Override
+ public void removeByKeyPrefix(String preFix, ContextType csType) {
+ List<ContextKey> removeKeys = getContextValueMapSet().findByKeyPrefix(preFix, csType);
+ if (CollectionUtils.isNotEmpty(removeKeys)){
+ for (ContextKey key : removeKeys){
+ remove(key);
+ }
+ logger.warn("Remove keyValue by key preFix{} and csType{} ", preFix, csType);
+ }
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/guava/ContextIDCacheLoader.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/guava/ContextIDCacheLoader.java
new file mode 100644
index 0000000..d081e57
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/guava/ContextIDCacheLoader.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.guava;
+
+import com.google.common.cache.CacheLoader;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValueGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ContextIDCacheLoader extends CacheLoader<String, ContextIDValue> {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextIDCacheLoader.class);
+
+ @Autowired
+ private ContextIDValueGenerator contextIDValueGenerator;
+
+ @Override
+ public ContextIDValue load(String contextID) throws Exception {
+ logger.info("Start to load contextID:{}", contextID);
+ ContextIDValue contextIDValue = contextIDValueGenerator.createContextIDValue(contextID);
+
+ logger.info("Finished to load contextID:{}", contextID);
+ return contextIDValue;
+ }
+}
+*/
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/guava/ContextIDRemoveListener.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/guava/ContextIDRemoveListener.java
new file mode 100644
index 0000000..d6889b3
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cache/guava/ContextIDRemoveListener.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cache.guava;
+
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import com.webank.wedatasphere.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import static com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType.ACCESS;
+
+@Component
+public class ContextIDRemoveListener implements RemovalListener<String, ContextIDValue> {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextIDRemoveListener.class);
+
+
+ ContextAsyncListenerBus listenerBus = DefaultContextListenerManager.getInstance().getContextAsyncListenerBus();
+
+
+ @Override
+ public void onRemoval(RemovalNotification<String, ContextIDValue> removalNotification) {
+ ContextIDValue value = removalNotification.getValue();
+ String contextIDStr = removalNotification.getKey();
+ if (StringUtils.isBlank(contextIDStr) || null == value || null == value.getContextID() ){
+ return;
+ }
+ logger.info("Start to remove ContextID({}) from cache", contextIDStr);
+ DefaultContextIDEvent defaultContextIDEvent = new DefaultContextIDEvent();
+ defaultContextIDEvent.setContextID(value.getContextKeyValueContext().getContextID());
+ defaultContextIDEvent.setOperateType(ACCESS);
+ listenerBus.post(defaultContextIDEvent);
+ logger.info("Finished to remove ContextID({}) from cache", contextIDStr);
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cleaner/AUTOCleaner.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cleaner/AUTOCleaner.java
new file mode 100644
index 0000000..c6de1c0
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/cleaner/AUTOCleaner.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.cleaner;
+
+/**
+ * @author peacewong
+ * @date 2020/2/15 11:44
+ */
+public interface AUTOCleaner {
+
+ void triggerCleanup();
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/conf/ContextCacheConf.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/conf/ContextCacheConf.java
new file mode 100644
index 0000000..52418ce
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/conf/ContextCacheConf.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.conf;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 17:04
+ */
+public class ContextCacheConf {
+
+ public final static String KEYWORD_SCAN_PACKAGE = CommonVars.apply("wds.linkis.cs.keyword.scan.package","com.webank.wedatasphere.linkis.cs").getValue();
+ public final static String KEYWORD_SPLIT = CommonVars.apply("wds.linkis.cs.keyword.split",",").getValue();
+
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndex.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndex.java
new file mode 100644
index 0000000..0a30f95
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndex.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.index;
+
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/2/10 9:42
+ */
+public interface ContextInvertedIndex {
+
+ List<String> getContextKeys(String keyword);
+
+ boolean addValue(String keyword, String contextKey);
+
+ List<String> getContextKeys(List<String> keywords);
+
+ boolean remove(String keyword, String contextKey);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndexSet.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndexSet.java
new file mode 100644
index 0000000..594ae65
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndexSet.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.index;
+
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author peacewong
+ * @date 2020/2/10 17:27
+ */
+public interface ContextInvertedIndexSet {
+
+ ContextInvertedIndex getContextInvertedIndex(ContextType contextType);
+
+ boolean addValue(String keyword, ContextKey contextKey);
+
+ boolean addValue(String keyword, String contextKey, ContextType contextType);
+
+ boolean addKeywords(Set<String> keywords, String contextKey, ContextType contextType);
+
+ List<String> getContextKeys(String keyword, ContextType contextType);
+
+ boolean remove(String keyword, String contextKey, ContextType contextType);
+
+ ContextInvertedIndex removeAll(ContextType contextType);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndexSetImpl.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndexSetImpl.java
new file mode 100644
index 0000000..c31acbf
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/ContextInvertedIndexSetImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.index;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * @author peacewong
+ * @date 2020/2/10 19:54
+ */
+public class ContextInvertedIndexSetImpl implements ContextInvertedIndexSet{
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextInvertedIndexSetImpl.class);
+
+ private Map<String, ContextInvertedIndex> invertedIndexMap = new HashMap<>();
+
+ @Override
+ public ContextInvertedIndex getContextInvertedIndex(ContextType contextType) {
+ String csType = contextType.name();
+ if (! invertedIndexMap.containsKey(csType)){
+ synchronized (csType.intern()){
+ if (! invertedIndexMap.containsKey(csType)){
+ logger.info("For ContextType({}) init invertedIndex", csType);
+ invertedIndexMap.put(csType, new DefaultContextInvertedIndex());
+ }
+ }
+ }
+ return invertedIndexMap.get(csType);
+ }
+
+ @Override
+ public boolean addValue(String keyword, ContextKey contextKey) {
+ return addValue(keyword, contextKey.getKey(), contextKey.getContextType());
+ }
+
+ @Override
+ public boolean addValue(String keyword, String contextKey, ContextType contextType) {
+ return getContextInvertedIndex(contextType).addValue(keyword, contextKey);
+ }
+
+ @Override
+ public boolean addKeywords(Set<String> keywords, String contextKey, ContextType contextType) {
+ Iterator<String> iterator = keywords.iterator();
+ while (iterator.hasNext()){
+ addValue(iterator.next(), contextKey, contextType);
+ }
+ return true;
+ }
+
+ @Override
+ public List<String> getContextKeys(String keyword, ContextType contextType) {
+ return getContextInvertedIndex(contextType).getContextKeys(keyword);
+ }
+
+ @Override
+ public boolean remove(String keyword, String contextKey, ContextType contextType) {
+ return getContextInvertedIndex(contextType).remove(keyword, contextKey);
+ }
+
+ @Override
+ public ContextInvertedIndex removeAll(ContextType contextType) {
+ return invertedIndexMap.remove(contextType.name());
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/DefaultContextInvertedIndex.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/DefaultContextInvertedIndex.java
new file mode 100644
index 0000000..6c708a1
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/index/DefaultContextInvertedIndex.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.index;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.collections.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/2/10 9:45
+ */
+public class DefaultContextInvertedIndex implements ContextInvertedIndex {
+
+ /**
+ * TODO Added ContextKey scoring feature
+ */
+ Multimap<String, String> indexMultimap = HashMultimap.create();
+
+ @Override
+ public List<String> getContextKeys(String keyword) {
+ Collection<String> keywords = indexMultimap.get(keyword);
+ return new ArrayList<String>(keywords);
+ }
+
+ @Override
+ public boolean addValue(String keyword, String contextKey) {
+ return indexMultimap.put(keyword, contextKey);
+ }
+
+ @Override
+ public List<String> getContextKeys(List<String> keywords) {
+ if(CollectionUtils.isEmpty(keywords)) {
+ return null;
+ }
+ List<String> contextKeys = new ArrayList<>();
+ for(String keyword:keywords){
+ contextKeys.addAll(getContextKeys(keyword));
+ }
+ return contextKeys;
+ }
+
+ @Override
+ public boolean remove(String keyword, String contextKey) {
+ return indexMultimap.remove(keyword, contextKey);
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ClassIntrospector.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ClassIntrospector.java
new file mode 100644
index 0000000..c53901d
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ClassIntrospector.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+
+import sun.misc.Unsafe;
+
+/**
+ * @author peacewong
+ * @date 2020/2/16 14:41
+ */
+public class ClassIntrospector {
+
+ private static final Unsafe unsafe;
+ /**
+ * Size of any Object reference
+ */
+ private static final int objectRefSize;
+
+ static {
+ try {
+ Field field = Unsafe.class.getDeclaredField("theUnsafe");
+ field.setAccessible(true);
+ unsafe = (Unsafe) field.get(null);
+
+ // 可以通过Object[]数组得到oop指针究竟是压缩后的4个字节还是未压缩的8个字节
+ objectRefSize = unsafe.arrayIndexScale(Object[].class);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Sizes of all primitive values
+ */
+ private static final Map<Class<?>, Integer> primitiveSizes;
+
+ static {
+ primitiveSizes = new HashMap<Class<?>, Integer>(10);
+ primitiveSizes.put(byte.class, 1);
+ primitiveSizes.put(char.class, 2);
+ primitiveSizes.put(int.class, 4);
+ primitiveSizes.put(long.class, 8);
+ primitiveSizes.put(float.class, 4);
+ primitiveSizes.put(double.class, 8);
+ primitiveSizes.put(boolean.class, 1);
+ }
+
+ /**
+ * Get object information for any Java object. Do not pass primitives to
+ * this method because they will boxed and the information you will get will
+ * be related to a boxed version of your value.
+ *
+ * @param obj Object to introspect
+ * @return Object info
+ * @throws IllegalAccessException
+ */
+ public ObjectInfo introspect(final Object obj)
+ throws IllegalAccessException {
+ try {
+ return introspect(obj, null);
+ } finally { // clean visited cache before returning in order to make
+ // this object reusable
+ m_visited.clear();
+ }
+ }
+
+ // we need to keep track of already visited objects in order to support
+ // cycles in the object graphs
+ private IdentityHashMap<Object, Boolean> m_visited = new IdentityHashMap<Object, Boolean>(
+ 100);
+
+ private ObjectInfo introspect(final Object obj, final Field fld)
+ throws IllegalAccessException {
+ // use Field type only if the field contains null. In this case we will
+ // at least know what's expected to be
+ // stored in this field. Otherwise, if a field has interface type, we
+ // won't see what's really stored in it.
+ // Besides, we should be careful about primitives, because they are
+ // passed as boxed values in this method
+ // (first arg is object) - for them we should still rely on the field
+ // type.
+ boolean isPrimitive = fld != null && fld.getType().isPrimitive();
+ boolean isRecursive = false; // will be set to true if we have already
+ // seen this object
+ if (!isPrimitive) {
+ if (m_visited.containsKey(obj)) {
+ isRecursive = true;
+ }
+ m_visited.put(obj, true);
+ }
+
+ final Class<?> type = (fld == null || (obj != null && !isPrimitive)) ? obj
+ .getClass() : fld.getType();
+ int arraySize = 0;
+ int baseOffset = 0;
+ int indexScale = 0;
+ if (type.isArray() && obj != null) {
+ baseOffset = unsafe.arrayBaseOffset(type);
+ indexScale = unsafe.arrayIndexScale(type);
+ arraySize = baseOffset + indexScale * Array.getLength(obj);
+ }
+
+ final ObjectInfo root;
+ if (fld == null) {
+ root = new ObjectInfo("", type.getCanonicalName(), getContents(obj,
+ type), 0, getShallowSize(type), arraySize, baseOffset,
+ indexScale);
+ } else {
+ final int offset = (int) unsafe.objectFieldOffset(fld);
+ root = new ObjectInfo(fld.getName(), type.getCanonicalName(),
+ getContents(obj, type), offset, getShallowSize(type),
+ arraySize, baseOffset, indexScale);
+ }
+
+ if (!isRecursive && obj != null) {
+ if (isObjectArray(type)) {
+ // introspect object arrays
+ final Object[] ar = (Object[]) obj;
+ for (final Object item : ar) {
+ if (item != null) {
+ root.addChild(introspect(item, null));
+ }
+ }
+ } else {
+ for (final Field field : getAllFields(type)) {
+ if ((field.getModifiers() & Modifier.STATIC) != 0) {
+ continue;
+ }
+ field.setAccessible(true);
+ root.addChild(introspect(field.get(obj), field));
+ }
+ }
+ }
+
+ root.sort(); // sort by offset
+ return root;
+ }
+
+ // get all fields for this class, including all superclasses fields
+ private static List<Field> getAllFields(final Class<?> type) {
+ if (type.isPrimitive()) {
+ return Collections.emptyList();
+ }
+ Class<?> cur = type;
+ final List<Field> res = new ArrayList<Field>(10);
+ while (true) {
+ Collections.addAll(res, cur.getDeclaredFields());
+ if (cur == Object.class) {
+ break;
+ }
+ cur = cur.getSuperclass();
+ }
+ return res;
+ }
+
+ // check if it is an array of objects. I suspect there must be a more
+ // API-friendly way to make this check.
+ private static boolean isObjectArray(final Class<?> type) {
+ if (!type.isArray()) {
+ return false;
+ }
+ if (type == byte[].class || type == boolean[].class
+ || type == char[].class || type == short[].class
+ || type == int[].class || type == long[].class
+ || type == float[].class || type == double[].class) {
+ return false;
+ }
+ return true;
+ }
+
+ // advanced toString logic
+ private static String getContents(final Object val, final Class<?> type) {
+ if (val == null) {
+ return "null";
+ }
+ if (type.isArray()) {
+ if (type == byte[].class)
+ return Arrays.toString((byte[]) val);
+ else if (type == boolean[].class)
+ return Arrays.toString((boolean[]) val);
+ else if (type == char[].class)
+ return Arrays.toString((char[]) val);
+ else if (type == short[].class)
+ return Arrays.toString((short[]) val);
+ else if (type == int[].class)
+ return Arrays.toString((int[]) val);
+ else if (type == long[].class)
+ return Arrays.toString((long[]) val);
+ else if (type == float[].class)
+ return Arrays.toString((float[]) val);
+ else if (type == double[].class)
+ return Arrays.toString((double[]) val);
+ else
+ return Arrays.toString((Object[]) val);
+ }
+ return val.toString();
+ }
+
+ // obtain a shallow size of a field of given class (primitive or object
+ // reference size)
+ private static int getShallowSize(final Class<?> type) {
+ if (type.isPrimitive()) {
+ final Integer res = primitiveSizes.get(type);
+ return res != null ? res : 0;
+ } else {
+ return objectRefSize;
+ }
+ }
+}
\ No newline at end of file
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ContextCacheMetric.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ContextCacheMetric.java
new file mode 100644
index 0000000..32edcee
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ContextCacheMetric.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+/**
+ * @author peacewong
+ * @date 2020/2/14 15:17
+ */
+public interface ContextCacheMetric extends Metrtic {
+
+ int getUsedCount();
+
+ void addCount();
+
+
+ int getCachedCount();
+
+ void setCachedCount(int count);
+
+ long getCachedMemory();
+
+ void setCachedMemory(long memory);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ContextIDMetric.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ContextIDMetric.java
new file mode 100644
index 0000000..0ae24c2
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ContextIDMetric.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+/**
+ * @author peacewong
+ * @date 2020/2/12 17:01
+ */
+public interface ContextIDMetric extends Metrtic{
+
+ int getUsedCount();
+
+ void addCount();
+
+ long getMemory();
+
+ void setMemory(long memory);
+
+ long getCachedTime();
+
+ long getLastAccessTime();
+
+ void setLastAccessTime(long accessTime);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/DefaultContextCacheMetric.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/DefaultContextCacheMetric.java
new file mode 100644
index 0000000..ebf42d6
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/DefaultContextCacheMetric.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+/**
+ * @author peacewong
+ * @date 2020/2/16 19:15
+ */
+public class DefaultContextCacheMetric implements ContextCacheMetric {
+
+ private int usedCount;
+
+ private int cachedCount;
+
+ private long memory;
+
+
+ @Override
+ public int getUsedCount() {
+ return this.usedCount;
+ }
+
+ @Override
+ public void addCount() {
+ this.usedCount++ ;
+ }
+
+ @Override
+ public int getCachedCount() {
+ return this.cachedCount;
+ }
+
+ @Override
+ public void setCachedCount(int count) {
+ if (count < 0) {
+ count =0;
+ }
+ this.cachedCount = count;
+ }
+
+ @Override
+ public long getCachedMemory() {
+ return this.memory;
+ }
+
+ @Override
+ public void setCachedMemory(long memory) {
+ if (memory < 0){
+ memory = 0;
+ }
+ this.memory = memory;
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/DefaultContextIDMetric.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/DefaultContextIDMetric.java
new file mode 100644
index 0000000..84eb113
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/DefaultContextIDMetric.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+/**
+ * @author peacewong
+ * @date 2020/2/15 16:19
+ */
+public class DefaultContextIDMetric implements ContextIDMetric {
+
+ private int usedCount;
+
+ private long memory;
+
+ private long cachedTime = System.currentTimeMillis();
+
+ private long accessTime = System.currentTimeMillis();
+
+ @Override
+ public int getUsedCount() {
+ return this.usedCount;
+ }
+
+ @Override
+ public void addCount() {
+ this.usedCount++;
+ }
+
+ @Override
+ public long getMemory() {
+ return this.memory;
+ }
+
+ @Override
+ public void setMemory(long memory) {
+ if (memory < 0) {
+ memory = 0;
+ }
+ this.memory = memory;
+ }
+
+ @Override
+ public long getCachedTime() {
+ return this.cachedTime;
+ }
+
+ @Override
+ public long getLastAccessTime() {
+ return this.accessTime;
+ }
+
+ @Override
+ public void setLastAccessTime(long accessTime) {
+ this.accessTime = accessTime;
+ }
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/Metrtic.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/Metrtic.java
new file mode 100644
index 0000000..832e973
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/Metrtic.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+/**
+ * @author peacewong
+ * @date 2020/2/14 15:17
+ */
+public interface Metrtic {
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ObjectInfo.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ObjectInfo.java
new file mode 100644
index 0000000..ac703ec
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/ObjectInfo.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/2/16 14:37
+ */
+public class ObjectInfo {
+
+ /** Field name */
+ public final String name;
+ /** Field type name */
+ public final String type;
+ /** Field data formatted as string */
+ public final String contents;
+ /** Field offset from the start of parent object */
+ public final int offset;
+ /** Memory occupied by this field */
+ public final int length;
+ /** Offset of the first cell in the array */
+ public final int arrayBase;
+ /** Size of a cell in the array */
+ public final int arrayElementSize;
+ /** Memory occupied by underlying array (shallow), if this is array type */
+ public final int arraySize;
+ /** This object fields */
+ public final List<ObjectInfo> children;
+
+ public ObjectInfo(String name, String type, String contents, int offset, int length, int arraySize,
+ int arrayBase, int arrayElementSize)
+ {
+ this.name = name;
+ this.type = type;
+ this.contents = contents;
+ this.offset = offset;
+ this.length = length;
+ this.arraySize = arraySize;
+ this.arrayBase = arrayBase;
+ this.arrayElementSize = arrayElementSize;
+ children = new ArrayList<ObjectInfo>( 1 );
+ }
+
+ public void addChild( final ObjectInfo info )
+ {
+ if ( info != null ) {
+ children.add( info );
+ }
+ }
+
+ /**
+ * Get the full amount of memory occupied by a given object. This value may be slightly less than
+ * an actual value because we don't worry about memory alignment - possible padding after the last object field.
+ *
+ * The result is equal to the last field offset + last field length + all array sizes + all child objects deep sizes
+ * @return Deep object size
+ */
+ public long getDeepSize()
+ {
+ //return length + arraySize + getUnderlyingSize( arraySize != 0 );
+ return addPaddingSize(arraySize + getUnderlyingSize( arraySize != 0 ));
+ }
+
+ long size = 0;
+
+ private long getUnderlyingSize( final boolean isArray )
+ {
+ //long size = 0;
+ for ( final ObjectInfo child : children ){
+ size += child.arraySize + child.getUnderlyingSize( child.arraySize != 0 );
+ }
+ if ( !isArray && !children.isEmpty() ){
+ int tempSize = children.get( children.size() - 1 ).offset + children.get( children.size() - 1 ).length;
+ size += addPaddingSize(tempSize);
+ }
+
+ return size;
+ }
+
+ private static final class OffsetComparator implements Comparator<ObjectInfo>
+ {
+ @Override
+ public int compare( final ObjectInfo o1, final ObjectInfo o2 )
+ {
+ return o1.offset - o2.offset; //safe because offsets are small non-negative numbers
+ }
+ }
+
+ //sort all children by their offset
+ public void sort()
+ {
+ Collections.sort( children, new OffsetComparator() );
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder();
+ toStringHelper( sb, 0 );
+ return sb.toString();
+ }
+
+ private void toStringHelper( final StringBuilder sb, final int depth )
+ {
+ depth( sb, depth ).append("name=").append( name ).append(", type=").append( type )
+ .append( ", contents=").append( contents ).append(", offset=").append( offset )
+ .append(", length=").append( length );
+ if ( arraySize > 0 )
+ {
+ sb.append(", arrayBase=").append( arrayBase );
+ sb.append(", arrayElemSize=").append( arrayElementSize );
+ sb.append( ", arraySize=").append( arraySize );
+ }
+ for ( final ObjectInfo child : children )
+ {
+ sb.append( '\n' );
+ child.toStringHelper(sb, depth + 1);
+ }
+ }
+
+ private StringBuilder depth( final StringBuilder sb, final int depth )
+ {
+ for ( int i = 0; i < depth; ++i ) {
+ sb.append( "\t");
+ }
+ return sb;
+ }
+
+ private long addPaddingSize(long size){
+ if(size % 8 != 0){
+ return (size / 8 + 1) * 8;
+ }
+ return size;
+ }
+
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/SizeEstimator.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/SizeEstimator.java
new file mode 100644
index 0000000..f1ebd11
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/metric/SizeEstimator.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.metric;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author peacewong
+ * @date 2020/2/16 15:33
+ */
+public class SizeEstimator {
+
+ private static final Logger logger = LoggerFactory.getLogger(SizeEstimator.class);
+
+
+ private static ClassIntrospector classIntrospector = new ClassIntrospector();
+
+ public static Long estimate(Object obj) {
+ try {
+ if (obj == null){
+ return 0L;
+ }
+ ObjectInfo info = classIntrospector.introspect(obj);
+ return info.getDeepSize();
+ } catch (Throwable e) {
+ logger.info("estimate size failed", e);
+ }
+ return 0L;
+ }
+
+}
+
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/parser/ContextKeyValueParser.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/parser/ContextKeyValueParser.java
new file mode 100644
index 0000000..17460ae
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/parser/ContextKeyValueParser.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.parser;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+
+import java.util.Set;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 16:15
+ */
+public interface ContextKeyValueParser {
+
+ Set<String> parse(ContextKeyValue contextKeyValue);
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/parser/DefaultContextKeyValueParser.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/parser/DefaultContextKeyValueParser.java
new file mode 100644
index 0000000..93fd3e2
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/parser/DefaultContextKeyValueParser.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.parser;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.conf.ContextCacheConf;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 16:19
+ */
+@Component
+public class DefaultContextKeyValueParser implements ContextKeyValueParser {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultContextKeyValueParser.class);
+
+
+ private ObjectMapper jackson = BDPJettyServerHelper.jacksonJson();
+
+ @PostConstruct
+ private void init() {
+ logger.info("init keyValueParser");
+ }
+
+
+ @Override
+ public Set<String> parse(ContextKeyValue contextKeyValue) {
+ //先解析key
+ Set<String> keywordSet = new HashSet<>();
+ try {
+ if (contextKeyValue != null && contextKeyValue.getContextValue() != null && StringUtils.isNotBlank(contextKeyValue.getContextValue().getKeywords())){
+ String keywordObj = contextKeyValue.getContextValue().getKeywords();
+
+ try {
+ Set<String> keySet = jackson.readValue(keywordObj, new TypeReference<Set<String>>() {});
+ keywordSet.addAll(keySet);
+ } catch (Exception e) {
+ //TODO Delete later
+ logger.info("deal Exception", e);
+ String[] keywords = keywordObj.split(ContextCacheConf.KEYWORD_SPLIT);
+ for (String keyword: keywords){
+ keywordSet.add(keyword);
+ }
+ }
+ //TODO The contextKey default are keyword
+ keywordSet.add(contextKeyValue.getContextKey().getKey());
+ }
+ } catch (Exception e){
+ if (null != contextKeyValue && null != contextKeyValue.getContextKey() && StringUtils.isNotBlank(contextKeyValue.getContextKey().getKey())){
+ logger.error("Failed to parse keywords of " + contextKeyValue.getContextKey().getKey(), e);
+ } else {
+ logger.error("Failed to parse keywords of contextKey", e);
+ }
+
+ }
+ return keywordSet;
+ }
+
+}
diff --git a/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/utils/ContextCacheUtils.java b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/utils/ContextCacheUtils.java
new file mode 100644
index 0000000..bd1f61b
--- /dev/null
+++ b/contextservice/cs-cache/src/main/java/com/webank/wedatasphere/linkis/cs/contextcache/utils/ContextCacheUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.utils;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 17:31
+ */
+public class ContextCacheUtils {
+
+ public static Set<String> getString(String s, String regex) {
+
+ Set<String> keywords = new HashSet<>();
+ Pattern p = Pattern.compile(regex);
+ Matcher m = p.matcher(s);
+ while(m.find()) {
+ keywords.add(m.group());
+
+ }
+ return keywords;
+ }
+}
diff --git a/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/csid/TestContextID.java b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/csid/TestContextID.java
new file mode 100644
index 0000000..a2344cd
--- /dev/null
+++ b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/csid/TestContextID.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.test.csid;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 20:41
+ */
+public class TestContextID implements ContextID {
+
+ String contextID;
+
+ @Override
+ public String getContextId() {
+ return contextID;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextID = contextId;
+ }
+}
diff --git a/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKey.java b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKey.java
new file mode 100644
index 0000000..a52adca
--- /dev/null
+++ b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKey.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.test.keyword;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 16:32
+ */
+public class TestContextKey implements ContextKey {
+
+ private String key;
+
+ private String keywords;
+
+ @KeywordMethod
+ @Override
+ public String getKey() {
+ return this.key;
+ }
+
+ @Override
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public ContextType getContextType() {
+ return ContextType.METADATA;
+ }
+
+ @Override
+ public void setContextType(ContextType contextType) {
+
+ }
+
+ @Override
+ public ContextScope getContextScope() {
+ return ContextScope.PUBLIC;
+ }
+
+ @Override
+ public void setContextScope(ContextScope contextScope) {
+
+ }
+
+ @KeywordMethod(splitter = ",")
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+ @Override
+ public void setType(int type) {
+
+ }
+}
diff --git a/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKeyValue.java b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKeyValue.java
new file mode 100644
index 0000000..f279ca7
--- /dev/null
+++ b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKeyValue.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.test.keyword;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 16:46
+ */
+public class TestContextKeyValue implements ContextKeyValue {
+
+ private ContextKey contextKey;
+
+ private ContextValue contextValue;
+
+ @Override
+ public ContextKey getContextKey() {
+ return this.contextKey;
+ }
+
+ @Override
+ public void setContextKey(ContextKey contextKey) {
+ this.contextKey = contextKey;
+ }
+
+ @Override
+ public ContextValue getContextValue() {
+ return this.contextValue;
+ }
+
+ @Override
+ public void setContextValue(ContextValue contextValue) {
+ this.contextValue = contextValue;
+ }
+}
diff --git a/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKeyValueParser.java b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKeyValueParser.java
new file mode 100644
index 0000000..dab52e3
--- /dev/null
+++ b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextKeyValueParser.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.test.keyword;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 16:26
+ */
+public class TestContextKeyValueParser {
+
+ /* @Test
+ public void testParser() {
+
+ AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext("com.webank.wedatasphere.linkis.cs");
+ System.out.println("ioc容器加载完成");
+ ContextKeyValueParser contextKeyValueParser = context.getBean(ContextKeyValueParser.class);
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("key1");
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+
+ ContextValue contextValue1 = new TestContextValue();
+ contextValue1.setKeywords("keyword4-keyword5-keyword6");
+ contextValue1.setValue("hello,hello2");
+ ContextKeyValue contextKeyValue1 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey1);
+ contextKeyValue1.setContextValue(contextValue1);
+ List<ContextKeyValue> contextKeyValueList = new ArrayList<>();
+ contextKeyValueList.add(contextKeyValue1);
+ Set<String> parse = contextKeyValueParser.parse(contextKeyValue1);
+ parse.stream().forEach(System.out::println);
+ }
+
+ public List<ContextKeyValue> testGetKeyValues(){
+ List<ContextKeyValue> contextKeyValueList = new ArrayList<>();
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("key1");
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+
+ ContextValue contextValue1 = new TestContextValue();
+ contextValue1.setKeywords("keyword4-keyword5-keyword6");
+ contextValue1.setValue("hello,hello2");
+ ContextKeyValue contextKeyValue1 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey1);
+ contextKeyValue1.setContextValue(contextValue1);
+
+ contextKeyValueList.add(contextKeyValue1);
+
+ ContextKey contextKey2 = new TestContextKey();
+ contextKey2.setKey("key2");
+ contextKey2.setKeywords("keywordd1,keywordd2,keywordd3");
+
+ ContextValue contextValue2 = new TestContextValue();
+ contextValue1.setKeywords("keywordd4-keywordd5-keywordd6");
+ contextValue1.setValue("hello,hello2");
+ ContextKeyValue contextKeyValue2 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey2);
+ contextKeyValue1.setContextValue(contextValue2);
+ contextKeyValueList.add(contextKeyValue2);
+ return contextKeyValueList;
+ }*/
+}
diff --git a/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextValue.java b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextValue.java
new file mode 100644
index 0000000..65e53df
--- /dev/null
+++ b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/keyword/TestContextValue.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.test.keyword;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ValueBean;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 16:44
+ */
+public class TestContextValue implements ContextValue {
+
+ private Object value;
+
+ private String keywords;
+
+ @KeywordMethod(splitter = "-")
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+
+ @KeywordMethod(regex = "hello")
+ @Override
+ public Object getValue() {
+ return this.value;
+ }
+
+ @Override
+ public void setValue(Object value) {
+ this.value = value;
+ }
+}
diff --git a/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/service/TestContextCacheService.java b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/service/TestContextCacheService.java
new file mode 100644
index 0000000..3414c3f
--- /dev/null
+++ b/contextservice/cs-cache/src/test/java/com/webank/wedatasphere/linkis/cs/contextcache/test/service/TestContextCacheService.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.contextcache.test.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.test.csid.TestContextID;
+import com.webank.wedatasphere.linkis.cs.contextcache.test.keyword.TestContextKey;
+import com.webank.wedatasphere.linkis.cs.contextcache.test.keyword.TestContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.test.keyword.TestContextValue;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 19:58
+ */
+public class TestContextCacheService {
+
+ AnnotationConfigApplicationContext context;
+
+ private String contextIDStr = "84693";
+
+ private ContextCacheService contextCacheService;
+
+ @Before
+ public void generateData() throws CSErrorException {
+ context = new AnnotationConfigApplicationContext("com.webank.wedatasphere.linkis.cs","com.webank.wedatasphere.linkis.mybatis");
+ System.out.println("ioc容器加载完成");
+ contextCacheService = context.getBean(ContextCacheService.class);
+ /* ContextPersistenceManager persistenceManager = context.getBean(ContextPersistenceManager.class);
+ persistenceManager.getContextIDPersistence().deleteContextID(contextIDStr);
+ PersistenceContextID persistenceContextID = new PersistenceContextID();
+ persistenceContextID.setContextId(String.valueOf(contextIDStr));
+ persistenceContextID.setUser("johnnwang");
+ persistenceContextID.setExpireTime(new Date());
+ persistenceContextID.setExpireType(ExpireType.TODAY);
+ persistenceContextID.setInstance("updateInstance");
+ persistenceContextID.setBackupInstance("updatebackup");
+ persistenceContextID.setApplication("hive");
+ persistenceManager.getContextIDPersistence().createContextID(persistenceContextID);
+ ContextID contextID = persistenceManager.getContextIDPersistence().getContextID(persistenceContextID.getContextId());
+ System.out.println(contextID.getContextId());
+
+ TestContextKey contextKey = new TestContextKey();
+ contextKey.setContextScope(ContextScope.FRIENDLY);
+ contextKey.setContextType(ContextType.OBJECT);
+ contextKey.setKey("flow1.node1.test");
+ contextKey.setKeywords("flow1,flow2,flow3");
+ TestContextValue contextValue = new TestContextValue();
+ contextValue.setKeywords("test1,test2,test3");
+ contextValue.setValue("test1.flow1");
+ TestContextKeyValue testContextKeyValue = new TestContextKeyValue();
+ testContextKeyValue.setContextKey(contextKey);
+ testContextKeyValue.setContextValue(contextValue);
+ persistenceManager.getContextMapPersistence().create(contextID, testContextKeyValue);*/
+ }
+
+ //@Test
+ public void testGetAll() {
+ try {
+ ContextID contextID = new TestContextID();
+ contextID.setContextId(contextIDStr);
+ List<ContextKeyValue> all = contextCacheService.getAll(contextID);
+ if (null != all){
+ all.stream().forEach(contextKeyValue -> {
+ System.out.println(contextKeyValue.getContextKey().getKey());
+ System.out.println(contextKeyValue.getContextValue().getValue());
+ });
+ }
+ } catch (Exception e){
+ e.printStackTrace();
+ }
+
+ }
+
+
+
+ // @Test
+ public void testGet() {
+ ContextID contextID = new TestContextID();
+ contextID.setContextId(contextIDStr);
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("flow1.node1.test");
+ ContextKeyValue contextKeyValue1 = contextCacheService.get(contextID, contextKey1);
+ System.out.println(contextKeyValue1.getContextValue().getValue());
+ }
+
+ // @Test
+ public void testGetValues() {
+ ContextID contextID = new TestContextID();
+ contextID.setContextId(contextIDStr);
+ List<ContextKeyValue> contextKeyValueList = contextCacheService.getValues(contextID, "flow2", ContextType.METADATA);
+ if (null != contextKeyValueList){
+ contextKeyValueList.stream().forEach(contextKeyValue -> {
+ System.out.println(contextKeyValue.getContextKey().getKey());
+ });
+ }
+
+
+ }
+
+ // @Test
+ public void testLike() {
+ ContextID contextID = new TestContextID();
+ contextID.setContextId(contextIDStr);
+ List<ContextKeyValue> contextKeyValueList = contextCacheService.getAllLikes(contextID, ".*node1.*", ContextType.METADATA);
+ if (null != contextKeyValueList){
+ contextKeyValueList.stream().forEach(contextKeyValue -> {
+ System.out.println(contextKeyValue.getContextKey().getKey());
+ });
+ }
+
+
+ }
+
+ // @Before
+ public void testPut() throws CSErrorException {
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId(contextIDStr);
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("key1");
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue1 = contextCacheService.get(contextID, contextKey1);
+
+ ContextKey contextKey3 = new TestContextKey();
+ contextKey3.setKey("key3");
+ contextKey3.setKeywords("keyworddd1,keyworddd2,keyworddd3");
+
+ ContextValue contextValue3 = new TestContextValue();
+ contextValue3.setKeywords("keyworddd4-keyworddd5-keyworddd6");
+ contextValue3.setValue("hello,hello3");
+ ContextKeyValue contextKeyValue3 = new TestContextKeyValue();
+ contextKeyValue3.setContextKey(contextKey3);
+ contextKeyValue3.setContextValue(contextValue3);
+ contextCacheService.put(contextID, contextKeyValue3);
+
+ ContextKeyValue contextKeyValue4 = contextCacheService.get(contextID, contextKey3);
+ System.out.println(contextKeyValue4.getContextKey().getKey());
+ }
+}
diff --git a/contextservice/cs-cache/src/test/resources/linkis.properties b/contextservice/cs-cache/src/test/resources/linkis.properties
new file mode 100644
index 0000000..ce3f1ee
--- /dev/null
+++ b/contextservice/cs-cache/src/test/resources/linkis.properties
@@ -0,0 +1,35 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.test.mode=true
+
+wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/ide_gz_bdap_sit_01?characterEncoding=UTF-8
+wds.linkis.server.mybatis.datasource.username=
+wds.linkis.server.mybatis.datasource.password=
+
+
+wds.linkis.log.clear=true
+wds.linkis.server.version=v1
+
+##restful
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.cs.server.restful
+
+##mybatis
+wds.linkis.server.mybatis.mapperLocations=classpath*:com\\webank\\wedatasphere\\linkis\\cs\\persistence\\dao\\impl\\*.xml
+
+wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.linkis.cs.persistence.entity
+
+wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.linkis.cs.persistence.dao
diff --git a/contextservice/cs-cache/src/test/resources/log4j.properties b/contextservice/cs-cache/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a7e6854
--- /dev/null
+++ b/contextservice/cs-cache/src/test/resources/log4j.properties
@@ -0,0 +1,36 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
diff --git a/contextservice/cs-cache/src/test/resources/log4j2.xml b/contextservice/cs-cache/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..2a9e19f
--- /dev/null
+++ b/contextservice/cs-cache/src/test/resources/log4j2.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Copyright 2019 WeBank
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/contextservice/cs-client/pom.xml b/contextservice/cs-client/pom.xml
new file mode 100644
index 0000000..eff02ce
--- /dev/null
+++ b/contextservice/cs-client/pom.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-client</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-gateway-httpclient-support</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-storage</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-httpclient</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+<!-- <dependency>-->
+<!-- <groupId>com.webank.wedatasphere.linkis</groupId>-->
+<!-- <artifactId>linkis-cs-highavailable</artifactId>-->
+<!-- <version>${linkis.version}</version>-->
+<!-- </dependency>-->
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-listener</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+
+
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/AbstractContextClient.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/AbstractContextClient.java
new file mode 100644
index 0000000..525d28f
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/AbstractContextClient.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client;
+
+/**
+ * created by cooperyang on 2020/2/11
+ * Description:
+ */
+public abstract class AbstractContextClient implements ContextClient{
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/Context.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/Context.java
new file mode 100644
index 0000000..a670924
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/Context.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+
+import java.util.List;
+
+
+/**
+ * created by cooperyang on 2020/2/10
+ * Description:Context是一个接口让用户能够在微服务中进行对一个工作流的Context进行操作
+ * 他应该有一个默认的实现，比如LinkisWorkFlowContext
+ */
+public interface Context {
+
+ /**
+ * 每一个Context都应该有一个ContextID与之对应d
+ * @return contextID是该Context对应的ContextID
+ */
+ public ContextID getContextID();
+
+ /**
+ * 设置ContextID
+ * @param contextID contextID
+ */
+ public void setContextID(ContextID contextID);
+
+ /**
+ * 通过contextKey来获取对应的contextValue
+ * @param contextKey 是标识一个ContextValue的一个key值，比如资源文件的名称
+ * @return 返回key对应的value, 比如resourceid 和 version
+ */
+ public ContextValue getContextValue(ContextKey contextKey) throws ErrorException;
+
+ /**
+ * 设置contextKeyValue
+ * @param contextKeyAndValue 需要设置的ContextKeyValue
+ * @throws ErrorException 可能由于网络原因出现的
+ */
+
+ public void setContextKeyAndValue(ContextKeyValue contextKeyAndValue) throws ErrorException;
+
+
+
+ public void set(ContextKey contextKey, ContextValue contextValue) throws ErrorException;
+
+ public void setLocal(ContextKey contextKey, ContextValue contextValue);
+
+ public void setLocal(ContextKeyValue contextKeyValue);
+
+ /**
+ * todo 是通过ContextCondition进行搜索
+ * @return
+ */
+ public List<ContextKeyValue> searchContext(List<ContextType> contextTypes,
+ List<ContextScope> contextScopes,
+ List<String> contains,
+ List<String> regex) throws ErrorException;
+
+
+ public void reset(ContextKey contextKey) throws ErrorException;
+
+ public void reset() throws ErrorException;
+
+
+ public void remove(ContextKey contextKey) throws ErrorException;
+
+ public void removeAll()throws ErrorException;
+
+ public void onBind(ContextIDListener contextIDListener) throws ErrorException;
+
+ public void onBind(ContextKey contextKey, ContextKeyListener contextKeyListener) throws ErrorException;
+
+
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/ContextClient.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/ContextClient.java
new file mode 100644
index 0000000..3c3a3be
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/ContextClient.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+
+import java.io.Closeable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * created by cooperyang on 2020/2/10
+ * Description:
+ */
+public interface ContextClient extends Closeable {
+ /**
+ * 通过与cs-server进行交互生成一个工作流的Context
+ * 传入的信息应该有工程名 工作流名
+ * user是传入的用户名
+ * @return 用户可以使用的Context
+ */
+ @Deprecated
+ Context createContext(String projectName, String flowName, String use, Map<String, Object> params) throws ErrorException;
+
+
+ Context createContext(ContextID contextID) throws ErrorException;
+
+
+ /**
+ * 通过ContextID获取
+ * @param contextID
+ * @return
+ * @throws ErrorException
+ */
+ Context getContext(ContextID contextID) throws ErrorException;
+
+ Context getContext(String contextIDStr) throws ErrorException;
+
+ ContextValue getContextValue(ContextID contextID, ContextKey contextKey) throws ErrorException;
+
+
+
+ void update(ContextID contextID, ContextKey contextKey, ContextValue contextValue) throws ErrorException;
+
+
+ /**
+ * 通过contextID和contextKey进行对某一个contextKey进行reset
+ * @param contextID contextId
+ * @param contextKey contexKey
+ * @throws ErrorException 可能捕获的异常
+ */
+ void reset(ContextID contextID, ContextKey contextKey) throws ErrorException;
+
+
+ /**
+ * 将整个contextID所有的contextKey进行reset,这个会用在工作流实时执行之前的一个reset
+ * @param contextID contextID
+ * @throws ErrorException
+ */
+ void reset(ContextID contextID) throws ErrorException;
+
+
+
+
+
+ /**
+ * 删除的操作是为了能够将contextid下面的contextkey进行删除
+ * 如果contextKey是空的话，则全部删除
+ * @param contextID
+ * @param contextKey
+ * @throws ErrorException
+ */
+ void remove(ContextID contextID, ContextKey contextKey) throws ErrorException;
+
+ void setContextKeyValue(ContextID contextID, ContextKeyValue contextKeyValue) throws ErrorException;
+
+ void bindContextIDListener(ContextIDListener contextIDListener) throws ErrorException;
+
+ void bindContextKeyListener(ContextKeyListener contextKeyListener) throws ErrorException;
+
+ /**
+ * 通过各种condition搜索contextkeyValue
+ * @return 一个contextKeyValue数组
+ * @throws ErrorException 可能出现的error
+ */
+ List<ContextKeyValue> search(ContextID contextID, List<ContextType> contextTypes,
+ List<ContextScope> contextScopes,
+ List<String> contains,
+ List<String> regex) throws ErrorException;
+
+ /**
+ * 通过各种condition搜索contextkeyValue
+ * upstreamOnly 表示取跟当前节点上游中查询
+ * nodeName 获取该节点的关系节点
+ * num 获取的节点数量，1 返回最近的一个， 无穷大表示所有节点
+ * @return 一个contextKeyValue数组
+ * @throws ErrorException 可能出现的error
+ */
+ List<ContextKeyValue> search(ContextID contextID, List<ContextType> contextTypes,
+ List<ContextScope> contextScopes,
+ List<String> contains,
+ List<String> regex,
+ boolean upstreamOnly,
+ String nodeName,
+ int num,
+ List<Class> contextValueTypes) throws ErrorException;
+
+
+
+ void removeAllValueByKeyPrefixAndContextType(ContextID contextID, ContextType contextType, String keyPrefix) throws ErrorException;
+
+ void removeAllValueByKeyPrefix(ContextID contextID, String keyPrefix) throws ErrorException;
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisContext.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisContext.java
new file mode 100644
index 0000000..8360d7f
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisContext.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client;
+
+/**
+ * created by cooperyang on 2020/2/17
+ * Description:
+ */
+public abstract class LinkisContext implements Context{
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisWorkFlowContext.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisWorkFlowContext.java
new file mode 100644
index 0000000..c13169e
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisWorkFlowContext.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.*;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * created by cooperyang on 2020/2/17
+ * Description:
+ */
+public class LinkisWorkFlowContext extends LinkisContext{
+
+ private ContextID contextID;
+
+ private ContextClient contextClient;
+
+ private String user;
+
+
+ private Map<ContextKey, ContextValue> keyValues = new ConcurrentHashMap<>();
+
+ @Override
+ public ContextID getContextID() {
+ return this.contextID;
+ }
+
+ @Override
+ public void setContextID(ContextID contextID) {
+ this.contextID = contextID;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ @Override
+ public ContextValue getContextValue(ContextKey contextKey) throws ErrorException{
+ if(keyValues.containsKey(contextKey)){
+ return keyValues.get(contextKey);
+ }else{
+ return this.contextClient.getContextValue(this.contextID, contextKey);
+ }
+ }
+
+
+ @Override
+ public void setContextKeyAndValue(ContextKeyValue contextKeyValue) throws ErrorException{
+ //todo 这个地方是为了进行一次缓存 不过有待商榷
+ //this.setLocal(contextKeyValue);
+ this.contextClient.setContextKeyValue(this.contextID, contextKeyValue);
+ }
+
+ @Override
+ public void set(ContextKey contextKey, ContextValue contextValue) throws ErrorException{
+ setLocal(contextKey, contextValue);
+ ContextKeyValue contextKeyValue = new CommonContextKeyValue();
+ contextKeyValue.setContextKey(contextKey);
+ contextKeyValue.setContextValue(contextValue);
+ this.setContextKeyAndValue(contextKeyValue);
+ }
+
+
+ @Override
+ public void setLocal(ContextKey contextKey, ContextValue contextValue) {
+ this.keyValues.put(contextKey, contextValue);
+ }
+
+ @Override
+ public void setLocal(ContextKeyValue contextKeyValue) {
+ this.keyValues.put(contextKeyValue.getContextKey(), contextKeyValue.getContextValue());
+ }
+
+ @Override
+ public List<ContextKeyValue> searchContext(List<ContextType> contextTypes,
+ List<ContextScope> contextScopes,
+ List<String> contains,
+ List<String> regex) throws ErrorException {
+ return this.contextClient.search(contextID, contextTypes, contextScopes, contains, regex);
+ }
+
+
+ @Override
+ public void reset(ContextKey contextKey) throws ErrorException{
+ this.contextClient.reset(contextID, contextKey);
+ }
+
+ @Override
+ public void reset() throws ErrorException {
+ this.contextClient.reset(this.contextID);
+ }
+
+ @Override
+ public void remove(ContextKey contextKey) throws ErrorException{
+ this.contextClient.remove(contextID, contextKey);
+ }
+
+ @Override
+ public void removeAll()throws ErrorException {
+ this.contextClient.remove(contextID, null);
+ }
+
+ @Override
+ public void onBind(ContextIDListener contextIDListener) throws ErrorException{
+ contextIDListener.setContextID(this.contextID);
+ contextIDListener.setContext(this);
+ this.contextClient.bindContextIDListener(contextIDListener);
+ }
+
+ @Override
+ public void onBind(ContextKey contextKey, ContextKeyListener contextKeyListener) throws ErrorException {
+ contextKeyListener.setContextKey(contextKey);
+ contextKeyListener.setContext(this);
+ this.contextClient.bindContextKeyListener(contextKeyListener);
+ }
+
+ public ContextClient getContextClient() {
+ return contextClient;
+ }
+
+ public void setContextClient(ContextClient contextClient) {
+ this.contextClient = contextClient;
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientConfig.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientConfig.java
new file mode 100644
index 0000000..505bd67
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientConfig.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.builder;
+
+/**
+ * created by cooperyang on 2020/2/10
+ * Description:
+ */
+public abstract class ContextClientConfig {
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientFactory.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientFactory.java
new file mode 100644
index 0000000..0e45602
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.builder;
+
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.http.HttpContextClient;
+
+/**
+ * created by cooperyang on 2020/2/10
+ * Description:
+ */
+public class ContextClientFactory {
+
+
+ private static final ContextClientConfig DEFAULT_CONTEXT_CLIENT_CONFIG;
+
+ static{
+ DEFAULT_CONTEXT_CLIENT_CONFIG = new HttpContextClientConfig();
+ }
+
+ public static ContextClient getOrCreateContextClient(){
+ return getOrCreateContextClient(DEFAULT_CONTEXT_CLIENT_CONFIG);
+ }
+
+ public static ContextClient getOrCreateContextClient(ContextClientConfig contextClientConfig){
+ return HttpContextClient.getInstance(contextClientConfig);
+ }
+
+
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/HttpContextClientConfig.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/HttpContextClientConfig.java
new file mode 100644
index 0000000..ef9b914
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/HttpContextClientConfig.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.builder;
+
+import com.webank.wedatasphere.linkis.common.conf.Configuration;
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextClientConf;
+import com.webank.wedatasphere.linkis.httpclient.authentication.AuthenticationStrategy;
+import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig;
+import com.webank.wedatasphere.linkis.httpclient.config.ClientConfigBuilder;
+import com.webank.wedatasphere.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy;
+
+/**
+ * created by cooperyang on 2020/2/10
+ * Description: 以http的方式和cs-server进行交互的配置,包括http的诸多配置
+ */
+public class HttpContextClientConfig extends ContextClientConfig{
+
+
+ private ClientConfig clientConfig;
+
+
+ public HttpContextClientConfig(){
+ //初始化clientConfig
+ String gatewayUrl = Configuration.getGateWayURL();
+ AuthenticationStrategy authenticationStrategy = new TokenAuthenticationStrategy();
+ int maxConnection = 10;
+ int connectionTimeout = 3000;
+ int readTimeout = 10000;
+ clientConfig = ClientConfigBuilder.newBuilder().addUJESServerUrl(gatewayUrl).
+ connectionTimeout(connectionTimeout).discoveryEnabled(false).loadbalancerEnabled(false).
+ maxConnectionSize(maxConnection).retryEnabled(false).readTimeout(readTimeout)
+ .setAuthenticationStrategy(authenticationStrategy).setAuthTokenKey(ContextClientConf.CONTEXT_CLIENT_AUTH_KEY().getValue())
+ .setAuthTokenValue(ContextClientConf.CONTEXT_CLIENT_AUTH_VALUE().getValue()).build();
+
+ }
+
+
+
+ /**
+ *
+ * @return 返回一个的
+ */
+ public ClientConfig getClientConfig(){
+ return this.clientConfig;
+ }
+
+
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/RPCHttpContextClientConfig.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/RPCHttpContextClientConfig.java
new file mode 100644
index 0000000..0318e9f
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/RPCHttpContextClientConfig.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.builder;
+
+/**
+ * created by cooperyang on 2020/2/10
+ * Description:
+ */
+public class RPCHttpContextClientConfig {
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/ContextPostActionBuilder.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/ContextPostActionBuilder.java
new file mode 100644
index 0000000..55c0228
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/ContextPostActionBuilder.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.http;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created by v_wbjftang on 2020/3/4.
+ */
+public class ContextPostActionBuilder {
+
+ private final DefaultContextPostAction action;
+
+ private Map<String, Object> requestParams = new HashMap<String, Object>(4);
+ private Map<String, String> headerParams = new HashMap<>(2);
+
+ public ContextPostActionBuilder(String url) {
+ action = new DefaultContextPostAction(url);
+ }
+
+ public static ContextPostActionBuilder of(String url) {
+ return new ContextPostActionBuilder(url);
+ }
+
+ public ContextPostActionBuilder with(ContextID contextID) throws ErrorException {
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ requestParams.put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ return this;
+ }
+
+
+ public ContextPostActionBuilder with(String key, Object object) throws ErrorException {
+ requestParams.put(key, object);
+ return this;
+ }
+
+ public ContextPostActionBuilder addHeader(String key, String value) throws ErrorException {
+ headerParams.put(key, value);
+ return this;
+ }
+
+ /* public ContextPostActionBuilder with(ContextKey contextKey) {
+ return this;
+ }
+
+ public ContextPostActionBuilder with(ContextKeyValue keyValue) {
+ return this;
+ }
+
+ public ContextPostActionBuilder with(ContextValue value) {
+ return this;
+ }*/
+
+ public DefaultContextPostAction build() {
+ action.getRequestPayloads().putAll(requestParams);
+ action.getHeaders().putAll(headerParams);
+ return action;
+ }
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpContextClient.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpContextClient.java
new file mode 100644
index 0000000..30649ee
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpContextClient.java
@@ -0,0 +1,603 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.http;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.AbstractContextClient;
+import com.webank.wedatasphere.linkis.cs.client.Context;
+import com.webank.wedatasphere.linkis.cs.client.LinkisWorkFlowContext;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientConfig;
+import com.webank.wedatasphere.linkis.cs.client.builder.HttpContextClientConfig;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener;
+import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener;
+import com.webank.wedatasphere.linkis.cs.client.listener.HeartBeater;
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextClientConf;
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextServerHttpConf;
+import com.webank.wedatasphere.linkis.cs.client.utils.ExceptionHelper;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.*;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant;
+import com.webank.wedatasphere.linkis.cs.common.search.ContextSearchConditionMapBuilder;
+import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig;
+import com.webank.wedatasphere.linkis.httpclient.dws.DWSHttpClient;
+import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig;
+import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult;
+import com.webank.wedatasphere.linkis.httpclient.request.Action;
+import com.webank.wedatasphere.linkis.httpclient.response.Result;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * created by cooperyang on 2020/2/11
+ * Description: HttpContextClient是ContextClient的使用Http方式进行通信的具体实现
+ * 一般可以将其做成单例
+ */
+public class HttpContextClient extends AbstractContextClient {
+
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(HttpContextClient.class);
+
+ private DWSHttpClient dwsHttpClient;
+ private ContextClientConfig contextClientConfig;
+ private final String linkis_version = ContextClientConf.LINKIS_WEB_VERSION().getValue();
+
+ private final String name = "HttpContextClient";
+
+ private static HttpContextClient httpContextClient;
+
+ private HeartBeater heartBeater;
+
+
+
+
+ private HttpContextClient(){
+
+ }
+
+ private HttpContextClient(ContextClientConfig contextClientConfig){
+ //初始化dwsHttpClient
+ this.contextClientConfig = contextClientConfig;
+ if (contextClientConfig instanceof HttpContextClientConfig){
+ HttpContextClientConfig httpContextClientConfig = (HttpContextClientConfig)contextClientConfig;
+ ClientConfig clientConfig = httpContextClientConfig.getClientConfig();
+ DWSClientConfig dwsClientConfig = new DWSClientConfig(clientConfig);
+ dwsClientConfig.setDWSVersion(linkis_version);
+ dwsHttpClient = new DWSHttpClient(dwsClientConfig, name);
+ }
+ if ("true".equals(ContextClientConf.HEART_BEAT_ENABLED().getValue())){
+ this.heartBeater = new HttpHeartBeater(contextClientConfig);
+ heartBeater.start();
+ }
+ }
+
+
+ public static HttpContextClient getInstance(ContextClientConfig contextClientConfig){
+ if (httpContextClient == null){
+ synchronized (HttpContextClient.class){
+ if (httpContextClient == null){
+ httpContextClient = new HttpContextClient(contextClientConfig);
+ }
+ }
+ }
+ return httpContextClient;
+ }
+
+
+ @Override
+ @Deprecated
+ public Context createContext(String projectName, String flowName, String user, Map<String, Object> params) throws ErrorException{
+ ContextCreateAction contextCreateAction = new ContextCreateAction();
+ LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID();
+ contextID.setProject(projectName);
+ contextID.setFlow(flowName);
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ contextCreateAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ contextCreateAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextCreateAction);
+ }catch(Exception e) {
+ LOGGER.error("create context failed", e);
+ ExceptionHelper.throwErrorException(80015, "create context failed", e);
+ }
+ if (result instanceof ContextCreateResult){
+ ContextCreateResult contextCreateResult = (ContextCreateResult)result;
+ int status = contextCreateResult.getStatus();
+ if (status != 0){
+ String errMsg = contextCreateResult.getMessage();
+ LOGGER.error("create context for project {}, flow {} failed, msg is {}", projectName, flowName, errMsg);
+ throw new ErrorException(80014, "create context failed" + errMsg);
+ }else{
+ LinkisWorkFlowContext context = new LinkisWorkFlowContext();
+ Map<String, Object> map = contextCreateResult.getData();
+ contextID.setContextId(map.get("contextId").toString());
+ context.setContextID(contextID);
+ context.setContextClient(this);
+ context.setUser(user);
+ return context;
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+
+ @Override
+ public Context createContext(ContextID contextID) throws ErrorException {
+ ContextCreateAction contextCreateAction = new ContextCreateAction();
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ contextCreateAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ contextCreateAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextCreateAction);
+ }catch(Exception e) {
+ LOGGER.error("create context failed", e);
+ ExceptionHelper.throwErrorException(80015, "create context failed", e);
+ }
+ if (result instanceof ContextCreateResult){
+ ContextCreateResult contextCreateResult = (ContextCreateResult)result;
+ int status = contextCreateResult.getStatus();
+ if (status != 0){
+ String errMsg = contextCreateResult.getMessage();
+ LOGGER.error("create context failed, msg is {}", errMsg);
+ throw new ErrorException(80014, "create context failed" + errMsg);
+ }else{
+ LinkisWorkFlowContext context = new LinkisWorkFlowContext();
+ Map<String, Object> map = contextCreateResult.getData();
+ contextID.setContextId(map.get("contextId").toString());
+ context.setContextID(contextID);
+ context.setContextClient(this);
+ return context;
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public Context getContext(ContextID contextId) throws ErrorException{
+ LinkisWorkFlowContext context = new LinkisWorkFlowContext();
+ context.setContextID(contextId);
+ context.setContextClient(this);
+ return context;
+ }
+
+ @Override
+ public Context getContext(String contextIDStr) throws ErrorException {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ return getContext(contextID);
+ }
+
+ @Override
+ public ContextValue getContextValue(ContextID contextID, ContextKey contextKey) throws ErrorException {
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ String contextKeyStr = SerializeHelper.serializeContextKey(contextKey);
+ ContextGetValueAction contextGetValueAction = new ContextGetValueAction();
+ contextGetValueAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ contextGetValueAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ contextGetValueAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_KEY_STR, contextKeyStr);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextGetValueAction);
+ }catch(Exception e) {
+ LOGGER.error("get context value id: {} , key: {} failed", contextIDStr, contextKeyStr, e);
+ ExceptionHelper.throwErrorException(80015, "get context value failed", e);
+ }
+ if (result instanceof ContextGetValueResult){
+ ContextGetValueResult contextGetValueResult = (ContextGetValueResult)result;
+ int status = contextGetValueResult.getStatus();
+ if (status != 0){
+ String errMsg = contextGetValueResult.getMessage();
+ LOGGER.error("get context value id: {} , key: {} failed, msg is {}", contextIDStr, contextKeyStr, errMsg);
+ throw new ErrorException(80014, "create context failed" + errMsg);
+ }else{
+ Map<String, Object> map = contextGetValueResult.getData();
+ if( null == map || null == map.get("contextValue") ){
+ return null;
+ }
+ String contextValueStr = map.get("contextValue").toString();
+ return SerializeHelper.deserializeContextValue(contextValueStr);
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+
+ @Override
+ public void update(ContextID contextID, ContextKey contextKey, ContextValue contextValue) throws ErrorException {
+ String contextIdStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextID);
+ String contextKeyValueStr = SerializeHelper.SERIALIZE_HELPER.serialize(new CommonContextKeyValue(contextKey, contextValue));
+ ContextSetKeyValueAction contextSetKeyValueAction = new ContextSetKeyValueAction();
+ contextSetKeyValueAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr);
+ contextSetKeyValueAction.getRequestPayloads().put("contextID", contextIdStr);
+ contextSetKeyValueAction.getRequestPayloads().put("contextKeyValue", contextKeyValueStr);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextSetKeyValueAction);
+ }catch(Exception e) {
+ LOGGER.error("update context failed", e);
+ ExceptionHelper.throwErrorException(80015, "update context failed", e);
+ }
+ if (result instanceof ContextSetKeyValueResult){
+ ContextSetKeyValueResult contextSetKeyValueResult = (ContextSetKeyValueResult)result;
+ int status = contextSetKeyValueResult.getStatus();
+ if (status != 0){
+ String errMsg = contextSetKeyValueResult.getMessage();
+ LOGGER.error("调用客户端去更新contextId {} 失败, 返回的错误信息是 {} ", contextIdStr, errMsg);
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public void reset(ContextID contextID, ContextKey contextKey) throws ErrorException{
+ String contextIdStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextID);
+ String contextKeyStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextKey);
+ ContextResetValueAction contextResetValueAction = new ContextResetValueAction();
+ contextResetValueAction.getRequestPayloads().put("contextKey", contextKeyStr);
+ contextResetValueAction.getRequestPayloads().put("contextID", contextIdStr);
+ // contextResetValueAction.getParameters().put("contextId", contextID.getContextId());
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextResetValueAction);
+ }catch(Exception e) {
+ LOGGER.error("reset contextID {}, contextKey {} failed", contextIdStr, contextKeyStr, e);
+ ExceptionHelper.throwErrorException(80015, "reset context failed", e);
+ }
+ if (result instanceof ContextResetResult){
+ ContextResetResult contextResetResult = (ContextResetResult)result;
+ int status = contextResetResult.getStatus();
+ if (status != 0){
+ String errMsg = contextResetResult.getMessage();
+ LOGGER.error("调用客户端去reset contextId {}, contextKey {} 失败, 返回的错误信息是 {} ", contextIdStr, contextKeyStr,errMsg);
+ throw new ErrorException(80015, "reset contextID failed");
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public void reset(ContextID contextID) throws ErrorException {
+ String contextIdStr = SerializeHelper.serializeContextID(contextID);
+ ContextResetIDAction contextResetIDAction = new ContextResetIDAction();
+ contextResetIDAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr);
+ //contextResetIDAction.getParameters().put("contextId", contextID.getContextId());
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextResetIDAction);
+ }catch(Exception e) {
+ LOGGER.error("reset contextID {} failed", contextIdStr, e);
+ ExceptionHelper.throwErrorException(80015, "reset context failed", e);
+ }
+ if (result instanceof ContextResetIDResult){
+ ContextResetIDResult contextResetResult = (ContextResetIDResult)result;
+ int status = contextResetResult.getStatus();
+ if (status != 0){
+ String errMsg = contextResetResult.getMessage();
+ LOGGER.error("调用客户端去reset contextId {} 失败, 返回的错误信息是 {} ", contextIdStr,errMsg);
+ throw new ErrorException(80015, "reset contextID failed");
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public void remove(ContextID contextID, ContextKey contextKey) throws ErrorException{
+ String contextIdStr = SerializeHelper.serializeContextID(contextID);
+ String contextKeyStr = SerializeHelper.serializeContextKey(contextKey);
+ ContextRemoveAction contextRemoveAction = new ContextRemoveAction(contextIdStr, contextKeyStr);
+ contextRemoveAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr);
+ contextRemoveAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_KEY_STR, contextKeyStr);
+ contextRemoveAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr);
+ contextRemoveAction.getRequestPayloads().put("contextId", contextID.getContextId());
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextRemoveAction);
+ }catch(Exception e) {
+ LOGGER.error("remove context id {} context key {} failed", contextIdStr, contextIdStr, e);
+ ExceptionHelper.throwErrorException(80015, "remove context failed", e);
+ }
+ if (result instanceof ContextRemoveResult){
+ ContextRemoveResult contextRemoveResult = (ContextRemoveResult)result;
+ int status = contextRemoveResult.getStatus();
+ if (status != 0){
+ String errMsg = contextRemoveResult.getMessage();
+ LOGGER.error("remove context failed contextID {}, contextKey {} ", contextIdStr, contextKey.getKey());
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public void setContextKeyValue(ContextID contextID, ContextKeyValue contextKeyValue) throws ErrorException {
+ String contextIDStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextID);
+ String contextKeyValueStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextKeyValue);
+ ContextSetKeyValueAction action = new ContextSetKeyValueAction();
+ action.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ action.getRequestPayloads().put("contextID", contextIDStr);
+ action.getRequestPayloads().put("contextKeyValue", contextKeyValueStr);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(action);
+ }catch(Exception e) {
+ LOGGER.error("set value failed", e);
+ ExceptionHelper.throwErrorException(80015, "update context failed", e);
+ }
+ if (result instanceof ContextSetKeyValueResult){
+ ContextSetKeyValueResult contextSetKeyValueResult = (ContextSetKeyValueResult)result;
+ int status = contextSetKeyValueResult.getStatus();
+ if (status != 0){
+ String errMsg = contextSetKeyValueResult.getMessage();
+ LOGGER.error("set value failed {} ,err is {}", contextIDStr, errMsg);
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public void bindContextIDListener(ContextIDListener contextIDListener) throws ErrorException{
+ ContextID contextID = contextIDListener.getContextID();
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ ContextBindIDAction contextBindIDAction = new ContextBindIDAction();
+ contextBindIDAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ // contextBindIDAction.getParameters().put("contextId", contextID.getContextId());
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextBindIDAction);
+ }catch(Exception e) {
+ LOGGER.error("bind context id {} failed", contextIDStr, e);
+ ExceptionHelper.throwErrorException(80015, "bind context id failed", e);
+ }
+ if (result instanceof ContextBindIDResult){
+ ContextBindIDResult contextBindIDResult = (ContextBindIDResult)result;
+ int status = contextBindIDResult.getStatus();
+ if (status != 0){
+ String errMsg = contextBindIDResult.getMessage();
+ LOGGER.error("bind context id failed {} ,err is {}", contextIDStr, errMsg);
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public void bindContextKeyListener(ContextKeyListener contextKeyListener) throws ErrorException{
+ ContextID contextID = contextKeyListener.getContext().getContextID();
+ ContextKey contextKey =contextKeyListener.getContextKey();
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ String contextKeyStr = SerializeHelper.serializeContextKey(contextKey);
+ ContextBindKeyAction contextBindKeyAction = new ContextBindKeyAction();
+ contextBindKeyAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ contextBindKeyAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ contextBindKeyAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_KEY_STR, contextKeyStr);
+ //todo 这里要改一下source的来历
+ contextBindKeyAction.getRequestPayloads().put("source", name);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextBindKeyAction);
+ }catch(Exception e) {
+ LOGGER.error("bind context id {} context key {} failed", contextIDStr, contextKeyStr, e);
+ ExceptionHelper.throwErrorException(80015, "bind context key failed", e);
+ }
+ if (result instanceof ContextBindKeyResult){
+ ContextBindKeyResult contextBindKeyResult = (ContextBindKeyResult)result;
+ int status = contextBindKeyResult.getStatus();
+ if (status != 0){
+ String errMsg = contextBindKeyResult.getMessage();
+ LOGGER.error("bind context id {} context key {} failed ,err is {}", contextIDStr, contextKeyStr, errMsg);
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ }
+
+ @Override
+ public List<ContextKeyValue> search(ContextID contextID,
+ List<ContextType> contextTypes,
+ List<ContextScope> contextScopes,
+ List<String> contains,
+ List<String> regex) throws ErrorException {
+ return search(contextID, contextTypes, contextScopes, contains, regex, false, null, Integer.MAX_VALUE, null);
+ }
+
+ @Override
+ public List<ContextKeyValue> search(ContextID contextID,
+ List<ContextType> contextTypes,
+ List<ContextScope> contextScopes,
+ List<String> contains,
+ List<String> regex,
+ boolean upstream,
+ String nodeName,
+ int num,
+ List<Class> contextValueTypes) throws ErrorException {
+ ContextSearchConditionMapBuilder builder = ContextSearchConditionMapBuilder.newBuilder();
+ if (contextTypes != null){
+ contextTypes.forEach(builder::contextTypes);
+ }
+ if (contextScopes != null){
+ contextScopes.forEach(builder::contextScopes);
+ }
+ if (contains != null){
+ contains.forEach(builder::contains);
+ }
+ if (regex != null){
+ regex.forEach(builder::regex);
+ }
+ builder.nearest(nodeName, num, upstream);
+ if (contextValueTypes != null) {
+ contextValueTypes.forEach(builder::contextValueTypes);
+ }
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ ContextSearchContextAction contextSearchContextAction = new ContextSearchContextAction();
+ contextSearchContextAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr);
+ contextSearchContextAction.getRequestPayloads().put("condition", builder.build());
+ contextSearchContextAction.getRequestPayloads().put("contextID", contextIDStr);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextSearchContextAction);
+ }catch(Exception e) {
+ LOGGER.error("search condition failed", e);
+ ExceptionHelper.throwErrorException(80015, "search condition failed", e);
+ }
+ if (result instanceof ContextSearchResult){
+ ContextSearchResult contextSearchResult = (ContextSearchResult)result;
+ int status = contextSearchResult.getStatus();
+ if (status != 0){
+ String errMsg = contextSearchResult.getMessage();
+ LOGGER.error("search condition failed, err is {}", errMsg);
+ }else{
+ Map<String, Object> data = contextSearchResult.getData();
+ if(data.get("contextKeyValue") != null){
+ List<ContextKeyValue> retKvs = new ArrayList<>();
+ Object o = data.get("contextKeyValue");
+ List<String> list = (List<String>)o;
+ list.stream().map(s -> {
+ try{
+ return SerializeHelper.deserializeContextKeyValue(s);
+ }catch(ErrorException e){
+ LOGGER.error("failed to deserialize {} to a contextKeyValue", s, e);
+ return null;
+ }
+ }).filter(Objects::nonNull).forEach(retKvs::add);
+ return retKvs;
+ }
+ }
+ }else if (result != null){
+ LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName());
+ throw new ErrorException(80015, "result is not a correct type");
+ }else{
+ LOGGER.error("result is null");
+ throw new ErrorException(80015, "result is null");
+ }
+ return null;
+ }
+
+
+
+ @Override
+ public void removeAllValueByKeyPrefixAndContextType(ContextID contextID, ContextType contextType, String keyPrefix) throws ErrorException {
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ DefaultContextPostAction action = ContextPostActionBuilder.of(ContextServerHttpConf.removeAllValueByKeyPrefixAndContextTypeURL())
+ .with(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).with(ContextHTTPConstant.CONTEXT_KEY_TYPE_STR, contextType.toString())
+ .with(ContextHTTPConstant.CONTEXT_KEY_PREFIX_STR, keyPrefix).addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).build();
+ checkDWSResult(execute(action));
+ }
+
+ @Override
+ public void removeAllValueByKeyPrefix(ContextID contextID, String keyPrefix) throws ErrorException {
+ String contextIDStr = SerializeHelper.serializeContextID(contextID);
+ DefaultContextPostAction action = ContextPostActionBuilder.of(ContextServerHttpConf.removeAllValueByKeyPrefixURL())
+ .with(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).with(ContextHTTPConstant.CONTEXT_KEY_PREFIX_STR, keyPrefix)
+ .addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).build();
+ checkDWSResult(execute(action));
+ }
+
+ private Result execute(Action action) throws ErrorException {
+ try{
+ return dwsHttpClient.execute(action);
+ }catch(Exception e) {
+ LOGGER.error("execute failed", e);
+ ExceptionHelper.throwErrorException(80015, "execute failed", e);
+ }
+ return null;
+ }
+
+ private DWSResult checkDWSResult(Result result) throws CSErrorException {
+ if(result instanceof DWSResult){
+ int status = ((DWSResult) result).getStatus();
+ if (status != 0){
+ String errMsg = ((DWSResult) result).getMessage();
+ LOGGER.error("request failed, err is {}", errMsg);
+ throw new CSErrorException(80015,errMsg);
+ }else {
+ return (DWSResult)result;
+ }
+ }else {
+ throw new CSErrorException(80015,"resulet is not instance of DWSResult");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try{
+ LOGGER.info("client close");
+ if (null != this.dwsHttpClient){
+ this.dwsHttpClient.close();
+ this.heartBeater.close();
+ }
+ } catch (Exception e){
+ LOGGER.error("Failed to close httpContextClient", e);
+ throw new IOException(e);
+ }
+
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpHeartBeater.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpHeartBeater.java
new file mode 100644
index 0000000..08f6a17
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpHeartBeater.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.http;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.common.utils.Utils;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientConfig;
+import com.webank.wedatasphere.linkis.cs.client.builder.HttpContextClientConfig;
+import com.webank.wedatasphere.linkis.cs.client.listener.*;
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextClientConf;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.*;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean;
+import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent;
+import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig;
+import com.webank.wedatasphere.linkis.httpclient.dws.DWSHttpClient;
+import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig;
+import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult;
+import com.webank.wedatasphere.linkis.httpclient.response.Result;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * created by cooperyang on 2020/2/18
+ * Description: heartbeater类的作用是为了csclient能够和csserver进行每秒钟交互的一个类，从server中获取内容，
+ * 然后封装成事件投递到 事件总线，来让监听器进行消费
+ */
+public class HttpHeartBeater implements HeartBeater {
+
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(HttpHeartBeater.class);
+
+ private ContextClientListenerBus<ContextClientListener, Event> contextClientListenerBus =
+ ContextClientListenerManager.getContextClientListenerBus();
+
+ private final String name = "ContextClientHTTPHeatBeater";
+
+ //todo 要改成某一个微服务的标识
+ private final String client_source = "TestClient";
+
+
+ private DWSHttpClient dwsHttpClient;
+
+
+
+ public HttpHeartBeater(ContextClientConfig contextClientConfig){
+ if (contextClientConfig instanceof HttpContextClientConfig){
+ HttpContextClientConfig httpContextClientConfig = (HttpContextClientConfig)contextClientConfig;
+ ClientConfig clientConfig = httpContextClientConfig.getClientConfig();
+ DWSClientConfig dwsClientConfig = new DWSClientConfig(clientConfig);
+ dwsClientConfig.setDWSVersion(ContextClientConf.LINKIS_WEB_VERSION().getValue());
+ dwsHttpClient = new DWSHttpClient(dwsClientConfig, name);
+ }
+ }
+
+
+
+
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void heartBeat() {
+ ContextHeartBeatAction contextHeartBeatAction = new ContextHeartBeatAction(client_source);
+ contextHeartBeatAction.getRequestPayloads().put("source", client_source);
+ Result result = null;
+ try{
+ result = dwsHttpClient.execute(contextHeartBeatAction);
+ }catch(Exception e){
+ LOGGER.error("执行heartbeat出现失败", e);
+ return ;
+ }
+ if (result instanceof ContextHeartBeatResult){
+ ContextHeartBeatResult contextHeartBeatResult = (ContextHeartBeatResult)result;
+ Map<String,Object> data = contextHeartBeatResult.getData();
+ Object object = data.get("ContextKeyValueBean");
+ List<ContextKeyValueBean> kvBeans = new ArrayList<>();
+ if (object instanceof List){
+ List<Object> list = (List<Object>)object;
+ list.stream().
+ filter(Objects::nonNull).
+ map(Object::toString).
+ map(str -> {
+ try{
+ return SerializeHelper.deserializeContextKVBean(str);
+ }catch(ErrorException e){
+ return null;
+ }
+ }).filter(Objects::nonNull).forEach(kvBeans::add);
+ }
+ if (kvBeans.size() > 0){
+ dealCallBack(kvBeans);
+ }
+ }
+ }
+
+ @Override
+ public void dealCallBack(List<ContextKeyValueBean> kvs) {
+ for(ContextKeyValueBean kv : kvs){
+ //todo 先忽略掉contextIDEvent
+ ContextKeyValue contextKeyValue = new CommonContextKeyValue();
+ contextKeyValue.setContextKey(kv.getCsKey());
+ contextKeyValue.setContextValue(kv.getCsValue());
+ DefaultContextKeyEvent event = new DefaultContextKeyEvent();
+ event.setContextID(kv.getCsID());
+ event.setOperateType(OperateType.UPDATE);
+ event.setContextKeyValue(contextKeyValue);
+ contextClientListenerBus.post(event);
+ }
+ }
+
+
+ @Override
+ public void start() {
+ Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ heartBeat();
+ }
+ }, 0, 1, TimeUnit.HOURS);
+ }
+
+ @Override
+ public void close() throws IOException {
+ try{
+ if (null != this.dwsHttpClient){
+ this.dwsHttpClient.close();
+ }
+ } catch (Exception e){
+ LOGGER.error("Failed to close httpContextClient", e);
+ throw new IOException(e);
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientEvent.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientEvent.java
new file mode 100644
index 0000000..9847922
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientEvent.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+
+/**
+ * created by cooperyang on 2020/2/18
+ * Description:
+ */
+public interface ContextClientEvent extends Event {
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListener.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListener.java
new file mode 100644
index 0000000..09cf07c
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListener.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.common.listener.EventListener;
+
+/**
+ * created by cooperyang on 2020/2/11
+ * Description:
+ */
+public interface ContextClientListener extends EventListener {
+ void onContextCreated(Event event);
+ void onContextUpdated(Event event);
+ void onEvent(Event event);
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerBus.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerBus.java
new file mode 100644
index 0000000..359841d
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerBus.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.common.listener.ListenerBus;
+import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus;
+
+/**
+ * created by cooperyang on 2020/2/11
+ * Description:
+ */
+public class ContextClientListenerBus<L extends ContextClientListener, E extends Event> extends ListenerEventBus<L, E> {
+
+
+ private static final String NAME = "ContextClientListenerBus";
+
+ private static final int CAPACITY = 10;
+
+ private static final int THREAD_SIZE = 20;
+
+ private static final int MAX_FREE_TIME = 5000;
+
+ public ContextClientListenerBus(){
+ super(CAPACITY, NAME,THREAD_SIZE,MAX_FREE_TIME);
+ }
+
+
+
+ @Override
+ public void doPostEvent(L listener, E event) {
+ listener.onEvent(event);
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerManager.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerManager.java
new file mode 100644
index 0000000..6e00b9b
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerManager.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+
+/**
+ * created by cooperyang on 2020/2/11
+ * Description:
+ * Manager的作用是为了方便用户将的
+ */
+public class ContextClientListenerManager {
+
+ private static ContextClientListenerBus<ContextClientListener, Event> contextClientListenerBus;
+
+ public static ContextClientListenerBus<ContextClientListener, Event> getContextClientListenerBus(){
+ if (contextClientListenerBus == null){
+ synchronized (ContextClientListenerManager.class){
+ if (contextClientListenerBus == null){
+ contextClientListenerBus = new ContextClientListenerBus<ContextClientListener, Event>();
+ }
+ }
+ }
+ return contextClientListenerBus;
+ }
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextIDListener.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextIDListener.java
new file mode 100644
index 0000000..84b9924
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextIDListener.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.client.Context;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextIDEvent;
+
+
+/**
+ * created by cooperyang on 2020/2/17
+ * Description: 这个listener是用来监听contextID的，用户可以进行实现
+ */
+public abstract class ContextIDListener implements ContextClientListener{
+
+
+ private ContextID contextID;
+
+ private Context context;
+
+
+ public ContextIDListener(){
+
+ }
+
+ public ContextIDListener(ContextID contextID){
+ this.contextID = contextID;
+ }
+
+ public ContextID getContextID() {
+ return contextID;
+ }
+
+ public void setContextID(ContextID contextID) {
+ this.contextID = contextID;
+ }
+
+
+ public Context getContext() {
+ return context;
+ }
+
+ public void setContext(Context context) {
+ this.context = context;
+ }
+
+ @Override
+ public void onContextCreated(Event event) {
+
+ }
+
+ @Override
+ public void onContextUpdated(Event event) {
+
+ }
+
+ public abstract void onContextRemoved(Event event);
+
+
+ @Override
+ public void onEvent(Event event) {
+ if (event instanceof DefaultContextIDEvent){
+ DefaultContextIDEvent defaultContextKeyEvent = (DefaultContextIDEvent)event;
+ if (defaultContextKeyEvent.getContextID().equals(contextID)){
+ switch(defaultContextKeyEvent.getOperateType()){
+ case UPDATE : onContextUpdated(defaultContextKeyEvent);
+ break;
+ case CREATE: onContextCreated(defaultContextKeyEvent);break;
+ case REMOVE: onContextRemoved(defaultContextKeyEvent);break;
+ default: break;
+ }
+ }
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextKeyListener.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextKeyListener.java
new file mode 100644
index 0000000..d6e57d9
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextKeyListener.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.client.Context;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * created by cooperyang on 2020/2/18
+ * Description:一个微服务对contextKey的监听器
+ */
+public abstract class ContextKeyListener implements ContextClientListener{
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ContextKeyListener.class);
+
+ private ContextKey contextKey;
+
+ private Context context;
+
+ public ContextKeyListener(){
+
+ }
+
+ public ContextKeyListener(ContextKey contextKey){
+ this.contextKey = contextKey;
+ }
+
+ public ContextKey getContextKey() {
+ return contextKey;
+ }
+
+ public void setContextKey(ContextKey contextKey) {
+ this.contextKey = contextKey;
+ }
+
+ public Context getContext() {
+ return context;
+ }
+
+ public void setContext(Context context) {
+ this.context = context;
+ }
+
+ @Override
+ public void onContextUpdated(Event event) {
+ if (event instanceof DefaultContextKeyEvent){
+ context.setLocal(((DefaultContextKeyEvent) event).getContextKeyValue());
+ }
+ }
+
+ @Override
+ public void onEvent(Event event) {
+ if (event instanceof DefaultContextKeyEvent){
+ DefaultContextKeyEvent defaultContextKeyEvent = (DefaultContextKeyEvent)event;
+ if (defaultContextKeyEvent.getContextKeyValue().getContextKey().equals(contextKey)){
+ switch(defaultContextKeyEvent.getOperateType()){
+ case UPDATE:onContextUpdated(defaultContextKeyEvent);break;
+ case CREATE:onContextCreated(defaultContextKeyEvent);break;
+ default:break;
+ }
+ }
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextUpdateEvent.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextUpdateEvent.java
new file mode 100644
index 0000000..3218cd2
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextUpdateEvent.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+/**
+ * created by cooperyang on 2020/2/18
+ * Description:
+ */
+public class ContextUpdateEvent implements ContextClientEvent{
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/HeartBeater.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/HeartBeater.java
new file mode 100644
index 0000000..fce9002
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/HeartBeater.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.listener;
+
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean;
+
+import java.io.Closeable;
+import java.util.List;
+
+/**
+ * created by cooperyang on 2020/2/11
+ * Description:
+ */
+public interface HeartBeater extends Closeable {
+ /**
+ *
+ */
+ public void heartBeat();
+ public void dealCallBack(List<ContextKeyValueBean> kvs);
+
+ public void start();
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSMetaDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSMetaDataService.java
new file mode 100644
index 0000000..2ec5617
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSMetaDataService.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.common.io.MetaData;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/9
+ */
+public class CSMetaDataService implements MetaDataService {
+
+ private static final Logger logger = LoggerFactory.getLogger(CSMetaDataService.class);
+
+ private static CSMetaDataService csMetaDataService;
+
+ private CSMetaDataService() {
+
+ }
+
+ public static CSMetaDataService getInstance() {
+ if (null == csMetaDataService) {
+ synchronized (CSMetaDataService.class) {
+ if (null == csMetaDataService) {
+ csMetaDataService = new CSMetaDataService();
+ }
+ }
+ }
+ return csMetaDataService;
+ }
+
+ @Override
+ public Map<ContextKey, MetaData> getAllUpstreamMetaData(String contextIDStr, String nodeName) throws CSErrorException {
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return null;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ return DefaultSearchService.getInstance().searchUpstreamContextMap(contextID, nodeName, Integer.MAX_VALUE, MetaData.class);
+ } catch (ErrorException e) {
+ logger.error("Deserialize contextid error. contextID : " + contextIDStr + ", e ", e);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextid error. contextID : " + contextIDStr + ", e " + e.getDesc());
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeService.java
new file mode 100644
index 0000000..fd44141
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeService.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+/**
+ * @author peacewong
+ * @date 2020/3/21 19:17
+ */
+public interface CSNodeService {
+
+ void initNodeCSInfo(String contextIDStr, String ndeName);
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeServiceImpl.java
new file mode 100644
index 0000000..0c7e448
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeServiceImpl.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author peacewong
+ * @date 2020/3/21 19:18
+ */
+public class CSNodeServiceImpl implements CSNodeService{
+
+ private final static Logger logger = LoggerFactory.getLogger(CSNodeServiceImpl.class);
+
+ private SearchService searchService = DefaultSearchService.getInstance();
+
+ private static CSNodeService csNodeService;
+
+ private CSNodeServiceImpl() {
+
+ }
+
+ public static CSNodeService getInstance() {
+ if (null == csNodeService) {
+ synchronized (CSNodeServiceImpl.class) {
+ if (null == csNodeService) {
+ csNodeService = new CSNodeServiceImpl();
+ }
+ }
+ }
+ return csNodeService;
+ }
+
+ @Override
+ public void initNodeCSInfo(String contextIDStr, String ndeName) {
+
+ try {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ contextClient.removeAllValueByKeyPrefixAndContextType(contextID, ContextType.METADATA, CSCommonUtils.NODE_PREFIX + ndeName);
+ } catch (Exception e) {
+ logger.error("Failed to init node cs Info", e);
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResourceService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResourceService.java
new file mode 100644
index 0000000..837f5da
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResourceService.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable;
+import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/9
+ */
+public class CSResourceService implements ResourceService {
+
+ private static final Logger logger = LoggerFactory.getLogger(CSResourceService.class);
+
+ private static CSResourceService csResourceService;
+
+ private CSResourceService() {
+
+ }
+
+ public static CSResourceService getInstance() {
+ if (null == csResourceService) {
+ synchronized (CSResourceService.class) {
+ if (null == csResourceService) {
+ csResourceService = new CSResourceService();
+ }
+ }
+ }
+ return csResourceService;
+ }
+
+ @Override
+ public Map<ContextKey, BMLResource> getAllUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException {
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return null;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ return DefaultSearchService.getInstance().searchUpstreamContextMap(contextID, nodeName, Integer.MAX_VALUE, BMLResource.class);
+ } catch (ErrorException e) {
+ logger.error("Deserialize contextid error. contextID : " + contextIDStr + ", e ", e);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextid error. contextID : " + contextIDStr + ", e " + e.getDesc());
+ }
+ }
+
+ @Override
+ public List<BMLResource> getUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException {
+ List<BMLResource> rsList = new ArrayList<>();
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return rsList;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ if (null != contextID) {
+ rsList = DefaultSearchService.getInstance().searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, BMLResource.class);
+ }
+ return rsList;
+ } catch (ErrorException e) {
+ logger.error("Failed to get Resource: " + e.getMessage());
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextID error. contextIDStr : ", e);
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataService.java
new file mode 100644
index 0000000..f930374
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataService.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.data.CSResultData;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 20:00
+ */
+public interface CSResultDataService {
+
+ CSResultData getCSResultData(String contextIDStr, String contextKey)throws CSErrorException;
+
+ void putCSResultData(String contextIDStr, String contextKeyStr, CSResultData csResultData) throws CSErrorException;
+
+ List<CSResultData> getUpstreamCSResultData(String contextIDStr, String nodeName) throws CSErrorException;
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataServiceImpl.java
new file mode 100644
index 0000000..e646047
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataServiceImpl.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.data.CSResultData;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 20:23
+ */
+public class CSResultDataServiceImpl implements CSResultDataService{
+
+ private final static Logger logger = LoggerFactory.getLogger(CSResultDataServiceImpl.class);
+
+ private SearchService searchService = DefaultSearchService.getInstance();
+
+ private static CSResultDataService csResultDataService;
+
+ private CSResultDataServiceImpl() {
+
+ }
+
+ public static CSResultDataService getInstance() {
+ if (null == csResultDataService) {
+ synchronized (CSResultDataServiceImpl.class) {
+ if (null == csResultDataService) {
+ csResultDataService = new CSResultDataServiceImpl();
+ }
+ }
+ }
+ return csResultDataService;
+ }
+
+ @Override
+ public CSResultData getCSResultData(String contextIDStr, String contextKeyStr) throws CSErrorException {
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(contextKeyStr)) {
+ return null;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr);
+ return searchService.getContextValue(contextID, contextKey, CSResultData.class);
+ } catch (ErrorException e) {
+ logger.error("Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage());
+ logger.error("exception ", e);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage());
+ }
+ }
+
+ @Override
+ public void putCSResultData(String contextIDStr, String contextKeyStr, CSResultData csResultData) throws CSErrorException {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr);
+ ContextValue contextValue = new CommonContextValue();
+ contextValue.setValue(csResultData);
+ contextClient.update(contextID, contextKey, contextValue);
+ } catch (ErrorException e) {
+ logger.error("Deserialize error. e ", e);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize error. e : " + e.getDesc());
+ }
+ }
+
+ @Override
+ public List<CSResultData> getUpstreamCSResultData(String contextIDStr, String nodeName) throws CSErrorException {
+ List<CSResultData> rsList = new ArrayList<>();
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return rsList;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ if (null != contextID) {
+ rsList = searchService.searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, CSResultData.class);
+ }
+ return rsList;
+ } catch (ErrorException e) {
+ logger.error("Deserialize contextID error. contextIDStr : " + contextIDStr, e);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextID error. contextIDStr : " + contextIDStr + "e : " + e.getDesc());
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSTableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSTableService.java
new file mode 100644
index 0000000..ddbce74
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSTableService.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.*;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/9
+ */
+public class CSTableService implements TableService {
+
+ private final static Logger logger = LoggerFactory.getLogger(CSTableService.class);
+
+ private SearchService searchService = DefaultSearchService.getInstance();
+
+ private static CSTableService csTableService;
+
+ private CSTableService() {
+
+ }
+
+ public static CSTableService getInstance() {
+ if (null == csTableService) {
+ synchronized (CSTableService.class) {
+ if (null == csTableService) {
+ csTableService = new CSTableService();
+ }
+ }
+ }
+ return csTableService;
+ }
+
+ @Override
+ public CSTable getCSTable(ContextID contextID, ContextKey contextKey) throws CSErrorException {
+ if (null == contextID || null == contextKey) {
+ return null;
+ }
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID();
+ }
+ CSTable csTable = searchService.getContextValue(contextID, contextKey, CSTable.class);
+ return csTable;
+ }
+
+ @Override
+ public List<CSTable> getUpstreamTables(String contextIDStr, String nodeName) throws CSErrorException {
+ List<CSTable> rsList = new ArrayList<>();
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return rsList;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ if (null != contextID) {
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID();
+ }
+ rsList = searchService.searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, CSTable.class);
+ }
+ return rsList;
+ } catch (ErrorException e) {
+ logger.error("Deserialize contextID error. contextIDStr : " + contextIDStr);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "getUpstreamTables error ", e);
+ }
+ }
+
+ @Override
+ public CSTable getUpstreamSuitableTable(String contextIDStr, String nodeName, String keyword) throws CSErrorException {
+ CSTable csTable = null;
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return csTable;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ if (null != contextID) {
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID();
+ }
+ csTable = searchService.searchContext(contextID, keyword, nodeName, CSTable.class);
+ }
+ } catch (ErrorException e) {
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "getUpstreamSuitableTable error ", e);
+ }
+ return csTable;
+ }
+
+ @Override
+ public List<ContextKeyValue> searchUpstreamTableKeyValue(String contextIDStr, String nodeName) throws CSErrorException {
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID();
+ }
+ return searchService.searchUpstreamKeyValue(contextID, nodeName, Integer.MAX_VALUE, CSTable.class);
+ } catch (ErrorException e) {
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Failed to searchUpstreamTableKeyValue ", e);
+ }
+ }
+
+
+ @Override
+ public void putCSTable(String contextIDStr, String contextKeyStr, CSTable csTable) throws CSErrorException {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr);
+ ContextValue contextValue = new CommonContextValue();
+ // todo check keywords
+ contextValue.setKeywords("");
+ contextValue.setValue(csTable);
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID();
+ }
+ contextClient.update(contextID, contextKey, contextValue);
+ } catch (ErrorException e) {
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "putCSTable error ", e);
+ }
+ }
+
+ @Override
+ public CSTable getCSTable(String contextIDStr, String contextKeyStr) throws CSErrorException {
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(contextKeyStr)) {
+ return null;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr);
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID();
+ }
+ return getCSTable(contextID, contextKey);
+ } catch (ErrorException e) {
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "getCSTable error ", e);
+ }
+ }
+
+
+ @Override
+ public void registerCSTable(String contextIDStr, String nodeName, String alias, CSTable csTable) throws CSErrorException {
+
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return;
+ }
+ String tableName = "";
+ if (StringUtils.isNotBlank(alias)) {
+ tableName = CSCommonUtils.CS_TMP_TABLE_PREFIX + nodeName + "_" + alias;
+ } else {
+ for (int i = 1; i < 10; i++) {
+ String tmpTable = CSCommonUtils.CS_TMP_TABLE_PREFIX + nodeName + "_rs" + i;
+ try {
+ ContextKey contextKey = new CommonContextKey();
+ contextKey.setContextScope(ContextScope.PUBLIC);
+ contextKey.setContextType(ContextType.METADATA);
+ contextKey.setKey(CSCommonUtils.getTableKey(nodeName, tmpTable));
+ CSTable oldCsTable = getCSTable(contextIDStr, SerializeHelper.serializeContextKey(contextKey));
+ if (null == oldCsTable) {
+ tableName = tmpTable;
+ break;
+ }
+ } catch (Exception e) {
+ tableName = tmpTable;
+ logger.warn("Failed to build tmp tableName", e);
+ break;
+ }
+ }
+ }
+ try {
+ csTable.setName(tableName);
+ ContextKey contextKey = new CommonContextKey();
+ contextKey.setContextScope(ContextScope.PUBLIC);
+ contextKey.setContextType(ContextType.METADATA);
+ contextKey.setKey(CSCommonUtils.getTableKey(nodeName, tableName));
+ putCSTable(contextIDStr, SerializeHelper.serializeContextKey(contextKey), csTable);
+ } catch (ErrorException e) {
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Failed to register cs tmp table ", e);
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSVariableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSVariableService.java
new file mode 100644
index 0000000..479f836
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSVariableService.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.object.LinkisVariable;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/3/12 20:29
+ */
+public class CSVariableService implements VariableService{
+
+ private final static Logger logger = LoggerFactory.getLogger(CSVariableService.class);
+
+ private SearchService searchService = DefaultSearchService.getInstance();
+
+ private static CSVariableService csVariableService;
+
+ private CSVariableService(){
+
+ }
+
+ @Override
+ public List<LinkisVariable> getUpstreamVariables(String contextIDStr, String nodeName) throws CSErrorException{
+ List<LinkisVariable> rsList = new ArrayList<>();
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) {
+ return rsList;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ if (null != contextID) {
+ rsList = searchService.searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, LinkisVariable.class);
+ }
+ return rsList;
+ } catch (Throwable e) {
+ logger.error("Failed to get variable : " + contextIDStr, e);
+ // throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Failed to get variable : " + contextIDStr + "e : " + e.getMessage());
+ }
+ return rsList;
+ }
+
+ @Override
+ public void putVariable(String contextIDStr, String contextKeyStr, LinkisVariable linkisVariable) throws CSErrorException {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr);
+ ContextValue contextValue = new CommonContextValue();
+ contextValue.setValue(linkisVariable);
+ contextClient.update(contextID, contextKey, contextValue);
+ } catch (ErrorException e) {
+ logger.error("Deserialize error. e ");
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize error. e : ", e);
+ }
+ }
+
+ public static CSVariableService getInstance() {
+ if (null == csVariableService) {
+ synchronized (CSVariableService.class) {
+ if (null == csVariableService) {
+ csVariableService = new CSVariableService();
+ }
+ }
+ }
+ return csVariableService;
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkService.java
new file mode 100644
index 0000000..f70db7e
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkService.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.WorkType;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/21
+ */
+public interface CSWorkService {
+
+ void initContextServiceInfo(String contextIDStr, WorkType workType) throws CSErrorException;
+
+ void initContextServiceInfo(String contextIDStr, List<WorkType> workTypes) throws CSErrorException;
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkServiceImpl.java
new file mode 100644
index 0000000..7c16dfe
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkServiceImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.WorkType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/21
+ */
+public class CSWorkServiceImpl implements CSWorkService {
+
+ private static final Logger logger = LoggerFactory.getLogger(CSWorkServiceImpl.class);
+ private CSWorkServiceImpl() {}
+ private static CSWorkService csWorkService = null;
+
+ public static CSWorkService getInstance() {
+ if (null == csWorkService) {
+ synchronized (CSWorkServiceImpl.class) {
+ if (null == csWorkService) {
+ csWorkService = new CSWorkServiceImpl();
+ }
+ }
+ }
+ return csWorkService;
+ }
+
+
+ @Override
+ public void initContextServiceInfo(String contextIDStr, WorkType workType) throws CSErrorException {
+ List<WorkType> typeList = new ArrayList<>();
+ typeList.add(workType);
+ initContextServiceInfo(contextIDStr, typeList);
+ }
+
+ @Override
+ public void initContextServiceInfo(String contextIDStr, List<WorkType> workTypes) throws CSErrorException {
+ try {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ for (WorkType type : workTypes) {
+ contextClient.removeAllValueByKeyPrefix(contextID, getWorkTypePrefix(type));
+ }
+ } catch (Exception e) {
+ logger.error("InitContextInfo error. contextIDStr : {}, workTypes : {}" + contextIDStr, CSCommonUtils.gson.toJson(workTypes));
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "InitContextInfo error. contextIDStr : " + contextIDStr + ", workTypes : " + CSCommonUtils.gson.toJson(workTypes), e);
+ }
+ }
+
+ private String getWorkTypePrefix(WorkType workType) {
+ String prefix = null;
+ switch (workType) {
+ case WORKSPACE:
+ prefix = CSCommonUtils.WORKSPACE_PREFIX;
+ break;
+ case PROJECT:
+ prefix = CSCommonUtils.PROJECT_PREFIX;
+ break;
+ case FLOW:
+ prefix = CSCommonUtils.FLOW_PREFIX;
+ break;
+ case NODE:
+ prefix = CSCommonUtils.NODE_PREFIX;
+ break;
+ default:
+ logger.error("Invalid workType : {}", workType);
+ }
+ return prefix;
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/DefaultSearchService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/DefaultSearchService.java
new file mode 100644
index 0000000..3e60f53
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/DefaultSearchService.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/6
+ */
+public class DefaultSearchService implements SearchService {
+ private static final Logger logger = LoggerFactory.getLogger(DefaultSearchService.class);
+ private static final Gson gson = new Gson();
+ private static SearchService searchService = null;
+
+ private DefaultSearchService() {
+ }
+
+ @Override
+ public <T> T getContextValue(ContextID contextID, ContextKey contextKey, Class<T> contextValueType) throws CSErrorException {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ ContextValue contextValue = null;
+ ContextType contextType = contextKey.getContextType();
+ try {
+ contextValue = contextClient.getContextValue(contextID, contextKey);
+ } catch (Exception e) {
+ logger.error("Failed to get ContextValue: " + e.getMessage());
+ throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "Failed to get ContextValue: ", e);
+ }
+
+ if (null == contextValue || null == contextValue.getValue()) {
+ return null;
+ } else if (contextValueType.isInstance(contextValue.getValue())) {
+ return (T) contextValue.getValue();
+ } else {
+ throw new CSErrorException(ErrorCode.INVALID_CONTEXT_VALUE_TYPE, "Invalid Context Type : " + contextType);
+ }
+ }
+
+ @Override
+ public <T> T searchContext(ContextID contextId, String keyword, String nodeName, Class<T> contextValueType) throws CSErrorException {
+ if (null == contextId || StringUtils.isBlank(contextId.getContextId()) || StringUtils.isBlank(nodeName) || null == contextValueType) {
+ logger.error("ContextID or nodeName or contextValueType cannot be null.");
+ throw new CSErrorException(ErrorCode.INVALID_NULL_STRING, "ContextID or nodeName or contextValueType cannot be null.");
+ }
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ List<String> contains = new ArrayList<>();
+ contains.add(keyword);
+ List<Class> contextValueTypes = new ArrayList<>();
+ contextValueTypes.add(contextValueType);
+ List<ContextKeyValue> contextKeyValues = null;
+ try {
+ contextKeyValues = contextClient.search(contextId, null, null, contains, null, false, nodeName, 1, contextValueTypes);
+ } catch (ErrorException e) {
+ logger.error("Search context value error.");
+ throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "Search context value error: ", e);
+ }
+ if (CollectionUtils.isEmpty(contextKeyValues) || null == contextKeyValues.get(0).getContextValue()) {
+ return null;
+ }
+ if (contextValueType.isInstance(contextKeyValues.get(0).getContextValue().getValue())) {
+ return (T) contextKeyValues.get(0).getContextValue().getValue();
+ } else {
+ throw new CSErrorException(ErrorCode.SEARCH_CONTEXT_VALUE_ERROR, "Search value : " + gson.toJson(contextKeyValues.get(0)
+ + " is not instance of class : " + contextValueType.getName()));
+ }
+ }
+
+ @Override
+ public <T> List<T> searchUpstreamContext(ContextID contextID, String nodeName, int num, Class<T> contextValueType) throws CSErrorException {
+ Map<ContextKey, T> contextMap = searchUpstreamContextMap(contextID, nodeName, num, contextValueType);
+ if (null == contextMap || contextMap.size() < 1) {
+ return null;
+ }
+ List<T> retValues = new ArrayList<>(contextMap.size());
+ contextMap.entrySet().stream().filter(kv -> null != kv && null != kv.getValue()).map(kv -> kv.getValue()).forEach(retValues::add);
+ return retValues;
+ }
+
+ @Override
+ public <T> Map<ContextKey, T> searchUpstreamContextMap(ContextID contextID, String nodeName, int num, Class<T> contextValueType) throws CSErrorException {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ if (null == contextID || StringUtils.isBlank(contextID.getContextId())) {
+ return null;
+ }
+ List<Class> contextValueTypes = new ArrayList<>();
+ contextValueTypes.add(contextValueType);
+ List<ContextKeyValue> contextKeyValueList = null;
+ try {
+ contextKeyValueList = contextClient.search(contextID, null, null, null, null, true, nodeName, num, contextValueTypes);
+ } catch (ErrorException e) {
+ throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "searchUpstreamContextMap error ", e);
+ }
+ if (CollectionUtils.isEmpty(contextKeyValueList)) {
+ return null;
+ }
+ Map<ContextKey, T> retValues = new HashMap<>(contextKeyValueList.size());
+ contextKeyValueList.stream().filter(kv -> null != kv && null != kv.getContextValue()).forEach(kv -> retValues.put(kv.getContextKey(), (T) (kv.getContextValue().getValue())));
+ return retValues;
+ }
+
+ @Override
+ public <T> List<ContextKeyValue> searchUpstreamKeyValue(ContextID contextID, String nodeName, int num, Class<T> contextValueType) throws CSErrorException {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ if (null == contextID || StringUtils.isBlank(contextID.getContextId())) {
+ return null;
+ }
+ List<Class> contextValueTypes = new ArrayList<>();
+ contextValueTypes.add(contextValueType);
+ List<ContextKeyValue> contextKeyValueList = null;
+ try {
+ contextKeyValueList = contextClient.search(contextID, null, null, null, null, true, nodeName, num, contextValueTypes);
+ } catch (ErrorException e) {
+ throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "searchUpstreamKeyValue error ", e);
+ }
+ return contextKeyValueList;
+ }
+
+ public static SearchService getInstance() {
+ if (null == searchService) {
+ synchronized (DefaultSearchService.class) {
+ if (null == searchService) {
+ searchService = new DefaultSearchService();
+ }
+ }
+ }
+ return searchService;
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataService.java
new file mode 100644
index 0000000..b746fda
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataService.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.data.LinkisJobData;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 22:11
+ */
+public interface LinkisJobDataService {
+
+ LinkisJobData getLinkisJobData(String contextIDStr, String contextKey)throws CSErrorException;
+
+ void putLinkisJobData(String contextIDStr, String contextKeyStr, LinkisJobData linkisJobData) throws CSErrorException;
+
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataServiceImpl.java
new file mode 100644
index 0000000..57421fb
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataServiceImpl.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.ContextClient;
+import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory;
+import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper;
+import com.webank.wedatasphere.linkis.cs.common.entity.data.LinkisJobData;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 22:12
+ */
+public class LinkisJobDataServiceImpl implements LinkisJobDataService {
+
+ private final static Logger logger = LoggerFactory.getLogger(LinkisJobDataServiceImpl.class);
+
+ private SearchService searchService = DefaultSearchService.getInstance();
+
+ private static LinkisJobDataService linkisJobDataService;
+
+ private LinkisJobDataServiceImpl() {
+
+ }
+
+ public static LinkisJobDataService getInstance() {
+ if (null == linkisJobDataService) {
+ synchronized (LinkisJobDataServiceImpl.class) {
+ if (null == linkisJobDataService) {
+ linkisJobDataService = new LinkisJobDataServiceImpl();
+ }
+ }
+ }
+ return linkisJobDataService;
+ }
+
+ @Override
+ public LinkisJobData getLinkisJobData(String contextIDStr, String contextKeyStr) throws CSErrorException {
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(contextKeyStr)) {
+ return null;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr);
+ return searchService.getContextValue(contextID, contextKey, LinkisJobData.class);
+ } catch (ErrorException e) {
+ logger.error("Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage());
+ logger.error("exception ", e);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage());
+ }
+ }
+
+ @Override
+ public void putLinkisJobData(String contextIDStr, String contextKeyStr, LinkisJobData linkisJobData) throws CSErrorException {
+ ContextClient contextClient = ContextClientFactory.getOrCreateContextClient();
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr);
+ ContextValue contextValue = new CommonContextValue();
+ contextValue.setValue(linkisJobData);
+ contextClient.update(contextID, contextKey, contextValue);
+ } catch (ErrorException e) {
+ logger.error("Deserialize error. e ", e);
+ throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize error. e : " + e.getDesc());
+ }
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/MetaDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/MetaDataService.java
new file mode 100644
index 0000000..babe9b2
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/MetaDataService.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.common.io.MetaData;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.Map;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/9
+ */
+public interface MetaDataService {
+
+ /**
+ * 通过ContextID和NodeName，获取上游的所有Metadata数据
+ * @param contextIDStr
+ * @param nodeName
+ * @return
+ * @throws CSErrorException
+ */
+ Map<ContextKey, MetaData> getAllUpstreamMetaData(String contextIDStr, String nodeName) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/ResourceService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/ResourceService.java
new file mode 100644
index 0000000..01b4c98
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/ResourceService.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/9
+ */
+public interface ResourceService {
+
+ /**
+ * 通过ContextID和NodeName，获取上游的所有Resource数据
+ * @param contextIDStr
+ * @param nodeName
+ * @return
+ */
+ Map<ContextKey, BMLResource> getAllUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException;
+
+ List<BMLResource> getUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException;
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/SearchService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/SearchService.java
new file mode 100644
index 0000000..d734e41
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/SearchService.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/6
+ */
+public interface SearchService {
+
+ <T> T getContextValue(ContextID contextId, ContextKey contextKey, Class<T> contextValueType) throws CSErrorException;
+
+ /**
+ *
+ * 返回匹配条件中最合适的一个
+ * @param contetID LinkisHAFlowContextID实例
+ * @param keyword 包含的关键字
+ * @param contextValueType 返回的contextValue必须是该类型的实例
+ * @param nodeName 如果nodeName是null，搜寻全部的，如果不为空搜寻上游的
+ * @param <T>
+ * @return
+ * @throws CSErrorException
+ */
+ <T> T searchContext(ContextID contetID, String keyword, String nodeName, Class<T> contextValueType) throws CSErrorException;
+
+ <T> List<T> searchUpstreamContext(ContextID contextID, String nodeName, int num, Class<T> contextValueType) throws CSErrorException;
+
+ <T> Map<ContextKey, T> searchUpstreamContextMap(ContextID contextID, String nodeName, int num, Class<T> contextValueType) throws CSErrorException;
+
+ <T> List<ContextKeyValue> searchUpstreamKeyValue(ContextID contextID, String nodeName, int num, Class<T> contextValueType) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/TableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/TableService.java
new file mode 100644
index 0000000..db66d08
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/TableService.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/9
+ */
+public interface TableService {
+
+ /**
+ * 通过ContextKey获取一个确定的CSTable
+ *
+ * @param contextID
+ * @param contextKey
+ * @return
+ * @throws CSErrorException
+ */
+ CSTable getCSTable(ContextID contextID, ContextKey contextKey) throws CSErrorException;
+
+ /**
+ * 获取上游节点的所有表
+ *
+ * @param contextIDStr
+ * @param nodeName
+ * @return
+ * @throws CSErrorException
+ */
+ List<CSTable> getUpstreamTables(String contextIDStr, String nodeName) throws CSErrorException;
+
+ void putCSTable(String contextIDStr, String ContextKey, CSTable csTable) throws CSErrorException;
+
+ CSTable getCSTable(String contextIDStr, String contextKey) throws CSErrorException;
+
+ CSTable getUpstreamSuitableTable(String contextIDStr, String nodeName, String keyword) throws CSErrorException;
+
+ List<ContextKeyValue> searchUpstreamTableKeyValue(String contextIDStr, String nodeName) throws CSErrorException;
+
+ void registerCSTable(String contextIDStr, String nodeName, String alias, CSTable csTable) throws CSErrorException;
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/VariableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/VariableService.java
new file mode 100644
index 0000000..168b4f3
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/VariableService.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.object.LinkisVariable;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+
+/**
+ * @author peacewong
+ * @date 2020/3/12 20:28
+ */
+public interface VariableService {
+
+ List<LinkisVariable> getUpstreamVariables(String contextIDStr, String nodeName) throws CSErrorException;
+
+ void putVariable(String contextIDStr, String contextKey, LinkisVariable linkisVariable) throws CSErrorException;
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ContextServiceUtils.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ContextServiceUtils.java
new file mode 100644
index 0000000..c106aea
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ContextServiceUtils.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.utils;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService;
+import com.webank.wedatasphere.linkis.cs.client.service.SearchService;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CombinedNodeIDContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * @author peacewong
+ * @date 2020/3/7 22:04
+ */
+public class ContextServiceUtils {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextServiceUtils.class);
+
+ /**
+ * TODO get instances
+ */
+ private static SearchService commonSearchService = DefaultSearchService.getInstance();
+
+ public static String getContextIDStrByMap(Map<String, Object> map){
+ String contextIDStr = null;
+ if (null != map){
+ Object value = map.get(CSCommonUtils.CONTEXT_ID_STR);
+ if (null != value){
+ contextIDStr = value.toString();
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ //commonSearchService.getContextValue()
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextIDStr = SerializeHelper.serializeContextID(((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID());
+ }
+ } catch (ErrorException e) {
+ logger.info("Failed to deserializeContextID", e);
+ }
+ }
+ }
+ return contextIDStr;
+ }
+
+
+ public static String getNodeNameStrByMap(Map<String, Object> map){
+ Object contextIDValue = map.get(CSCommonUtils.CONTEXT_ID_STR);
+ if (null == contextIDValue) {
+ return null;
+ }
+
+ String nodeName = null;
+ try {
+ if (null != map) {
+ Object value = map.get(CSCommonUtils.NODE_NAME_STR);
+ if (null != value) {
+ nodeName = value.toString();
+
+ }
+ }
+ if (StringUtils.isBlank(nodeName)) {
+ nodeName = getNodeNameByCombinedNodeIDContextID(contextIDValue.toString());
+ }
+ } catch (Exception e) {
+ logger.info("Failed to get nodeName", e);
+ }
+ map.put(CSCommonUtils.NODE_NAME_STR, nodeName);
+ return nodeName;
+ }
+
+
+
+ public static String getContextIDStrByProperties(Properties properties){
+ String contextIDStr = null;
+ if (null != properties){
+ Object value = properties.get(CSCommonUtils.CONTEXT_ID_STR);
+ if (null != value) {
+ contextIDStr = value.toString();
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ //commonSearchService.getContextValue()
+ if (contextID instanceof CombinedNodeIDContextID) {
+ contextIDStr = SerializeHelper.serializeContextID(((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID());
+ }
+ } catch (ErrorException e) {
+ logger.info("Failed to deserializeContextID", e);
+ }
+ }
+ }
+ return contextIDStr;
+ }
+
+ public static String getNodeNameStrByProperties(Properties properties) {
+ Object contextIDValue = properties.get(CSCommonUtils.CONTEXT_ID_STR);
+ if (null == contextIDValue) {
+ return null;
+ }
+ String nodeName = null;
+
+ if (null != properties) {
+ Object value = properties.get(CSCommonUtils.NODE_NAME_STR);
+ if (null != value) {
+ nodeName = value.toString();
+ }
+ }
+ if (StringUtils.isBlank(nodeName)) {
+ nodeName = getNodeNameByCombinedNodeIDContextID(contextIDValue.toString());
+ }
+ properties.put(CSCommonUtils.NODE_NAME_STR, nodeName);
+ return nodeName;
+ }
+
+
+ public static String getNodeNameByCombinedNodeIDContextID(String contextIDStr){
+ String nodeName = null;
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ //commonSearchService.getContextValue()
+ if (contextID instanceof CombinedNodeIDContextID){
+ logger.info("contextID{} is combinedNodeIDContextID", contextID.getContextId());
+ String nodeID = ((CombinedNodeIDContextID) contextID).getNodeID();
+ return getNodeNameByNodeID(SerializeHelper.serializeContextID(((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID()), nodeID);
+ }
+ } catch (Exception e) {
+ logger.info("Failed to get nodeName", e);
+ }
+ return nodeName;
+ }
+
+ public static String getNodeNameByNodeID(String contextIDStr, String nodeID) {
+ if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeID)){
+ return null;
+ }
+ try {
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ ContextKey contextKey = new CommonContextKey();
+ contextKey.setContextType(ContextType.OBJECT);
+ contextKey.setContextScope(ContextScope.PUBLIC);
+ contextKey.setKey(CSCommonUtils.FLOW_INFOS);
+ CSFlowInfos csFlowInfos = commonSearchService.getContextValue(contextID, contextKey, CSFlowInfos.class);
+ if (null != csFlowInfos && null != csFlowInfos.getInfos()) {
+ Object idName = csFlowInfos.getInfos().get(CSCommonUtils.ID_NODE_NAME);
+ if (null != idName) {
+ return ((Map<String, String>) idName).get(nodeID);
+ }
+ }
+ } catch (ErrorException e) {
+ logger.info("Failed to get nodeName ", e);
+ }
+ return null;
+ }
+
+ public static String createCombinedNodeIDContextID(String contextIDStr, String nodeID) throws ErrorException {
+ if (StringUtils.isNotBlank(contextIDStr) && StringUtils.isNotBlank(nodeID)){
+ ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr);
+ if (null != contextID){
+ return SerializeHelper.serializeContextID(new CombinedNodeIDContextID(contextID, nodeID));
+ }
+ }
+ return null;
+ }
+
+ /* public static String[] getContextIDAndNodeName(Map<String, Object> map){
+ String contextIDStr = getContextIDStrByMap(map);
+ if (StringUtils.isBlank(contextIDStr)) {
+ return null;
+ }
+
+ }*/
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ExceptionHelper.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ExceptionHelper.java
new file mode 100644
index 0000000..211469f
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ExceptionHelper.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.utils;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+
+/**
+ * created by cooperyang on 2020/2/19
+ * Description:
+ */
+public class ExceptionHelper {
+ public static void throwErrorException(int errCode, String errMsg, Throwable t)throws ErrorException {
+ ErrorException errorException = new ErrorException(errCode, errMsg);
+ errorException.initCause(t);
+ throw errorException;
+ }
+}
diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/SerializeHelper.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/SerializeHelper.java
new file mode 100644
index 0000000..51a100c
--- /dev/null
+++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/SerializeHelper.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.utils;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * created by cooperyang on 2020/2/23
+ * Description:
+ */
+public class SerializeHelper {
+
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SerializeHelper.class);
+
+ public static final SerializationHelper SERIALIZE_HELPER = ContextSerializationHelper.getInstance();
+
+ public static String serializeContextID(ContextID contextID) throws ErrorException{
+ return SERIALIZE_HELPER.serialize(contextID);
+ }
+
+ public static ContextID deserializeContextID(String contextIDStr) throws ErrorException{
+ return (ContextID) SERIALIZE_HELPER.deserialize(contextIDStr);
+ }
+
+
+
+ public static String serializeContextKey(ContextKey contextKey) throws ErrorException {
+ return SERIALIZE_HELPER.serialize(contextKey);
+ }
+
+
+
+ public static ContextKey deserializeContextKey(String contextKeyStr) throws ErrorException{
+ return (ContextKey)SERIALIZE_HELPER.deserialize(contextKeyStr);
+ }
+
+ public static String serializeContextValue(ContextValue contextValue) throws ErrorException{
+ return SERIALIZE_HELPER.serialize(contextValue);
+ }
+
+ public static ContextValue deserializeContextValue(String contextValueStr) throws ErrorException{
+ return (ContextValue)SERIALIZE_HELPER.deserialize(contextValueStr);
+ }
+
+ public static ContextKeyValueBean deserializeContextKVBean(String contextKVBeanStr) throws ErrorException{
+ return (ContextKeyValueBean)SERIALIZE_HELPER.deserialize(contextKVBeanStr);
+ }
+
+ public static ContextKeyValue deserializeContextKeyValue(String contextKVStr) throws ErrorException{
+ return (ContextKeyValue)SERIALIZE_HELPER.deserialize(contextKVStr);
+ }
+
+
+
+}
diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/exception/ProtocolNotMatchException.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/exception/ProtocolNotMatchException.scala
new file mode 100644
index 0000000..684822b
--- /dev/null
+++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/exception/ProtocolNotMatchException.scala
@@ -0,0 +1,24 @@
+/**
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.exception
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+
+/**
+ * created by cooperyang on 2020/2/18
+ * Description:
+ */
+case class ProtocolNotMatchException(errMsg:String) extends ErrorException(70059, errMsg)
diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextAction.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextAction.scala
new file mode 100644
index 0000000..d04dd12
--- /dev/null
+++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextAction.scala
@@ -0,0 +1,112 @@
+/**
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.http
+
+import com.webank.wedatasphere.linkis
+import com.webank.wedatasphere.linkis.cs.client.utils.{ContextClientConf, ContextClientUtils, ContextServerHttpConf}
+import com.webank.wedatasphere.linkis.httpclient.request.{GetAction, POSTAction, UserAction}
+import org.apache.commons.text.StringEscapeUtils
+/**
+ * created by cooperyang on 2020/2/11
+ * Description:
+ */
+trait ContextAction extends UserAction{
+
+ private var user:String = "hadoop"
+
+ override def setUser(user: String): Unit = this.user = user
+
+ override def getUser: String = this.user
+}
+
+abstract class ContextGETAction extends GetAction with ContextAction
+
+abstract class ContextPostAction extends POSTAction with ContextAction {
+ override def getRequestPayload: String = ContextClientUtils.gson.toJson(getRequestPayloads)
+}
+
+case class ContextCreateAction() extends ContextPostAction{
+
+ override def getURL: String = ContextServerHttpConf.createContextURL
+
+}
+
+case class ContextGetValueAction() extends ContextPostAction{
+ override def getURL: String = ContextServerHttpConf.getContextValueURL
+}
+
+
+case class ContextUpdateAction() extends ContextPostAction{
+
+ override def getURL: String = ContextServerHttpConf.updateContextURL
+
+}
+
+case class ContextSetKeyValueAction() extends ContextPostAction{
+
+ override def getURL: String = ContextServerHttpConf.setKeyValueURL
+
+}
+
+
+case class ContextResetValueAction() extends ContextPostAction{
+
+ override def getURL: String = ContextServerHttpConf.resetKeyValueURL
+
+}
+
+
+case class ContextResetIDAction() extends ContextPostAction{
+ override def getURL: String = ContextServerHttpConf.resetContextIdURL
+}
+
+
+case class ContextRemoveAction(contextId:String,
+ contextKey:String) extends ContextPostAction with UserAction {
+
+ override def getURL: String = ContextServerHttpConf.removeValueURL
+
+}
+
+
+case class ContextBindIDAction() extends ContextPostAction{
+ override def getURL: String = ContextServerHttpConf.onBindIDURL
+}
+
+case class ContextBindKeyAction() extends ContextPostAction{
+ override def getURL: String = ContextServerHttpConf.onBindKeyURL
+}
+
+
+
+
+case class ContextFetchAction(contextId:String) extends ContextGETAction{
+ override def getURL: String = ContextServerHttpConf.getContextIDURL
+}
+
+case class ContextHeartBeatAction(client:String) extends ContextPostAction {
+ override def getURL: String = ContextServerHttpConf.heartBeatURL
+}
+
+
+case class ContextSearchContextAction() extends ContextPostAction{
+ override def getURL: String = ContextServerHttpConf.searchURL
+}
+
+case class DefaultContextPostAction(url:String) extends ContextPostAction{
+ // TODO: 类太多了,放一个default
+ override def getURL: String = url
+}
\ No newline at end of file
diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextResult.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextResult.scala
new file mode 100644
index 0000000..09216ab
--- /dev/null
+++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextResult.scala
@@ -0,0 +1,128 @@
+/**
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.http
+
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean
+import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult
+import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult
+
+/**
+ * created by cooperyang on 2020/2/11
+ * Description:
+ */
+abstract class ContextResult extends DWSResult
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/createContextID")
+class ContextCreateResult extends ContextResult {
+ var contextId:String = _
+ def setContextId(contextId:String):Unit = this.contextId = contextId
+ def getContextId:String = this.contextId
+}
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/resetValue")
+class ContextResetResult extends ContextResult {
+
+}
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/setValueByKey")
+class ContextUpdateResult extends ContextResult {
+
+}
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/getContext")
+class ContextGetResult extends ContextResult {
+ var contextId:String = _
+ var contextKeyValues:java.util.Map[String, String] = _
+
+ def setContextId(contextId:String):Unit = this.contextId = contextId
+ def getContextId:String = this.contextId
+
+ def setContextKeyValues(kvs:java.util.Map[String,String]):Unit = this.contextKeyValues = kvs
+ def getContextKeyValues:java.util.Map[String, String] = this.contextKeyValues
+
+}
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/removeValue")
+class ContextRemoveResult extends ContextResult {
+
+}
+
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/heartbeat")
+class ContextHeartBeatResult extends ContextResult {
+ import java.util
+ var contextKeyValueBeans:util.List[ContextKeyValueBean] = _
+ def setContextKeyValueBeans(kvs:util.List[ContextKeyValueBean]):Unit = this.contextKeyValueBeans= kvs
+ def getContextKeyValueBeans:util.List[ContextKeyValueBean] = this.contextKeyValueBeans;
+}
+
+
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/setValue")
+class ContextSetKeyValueResult extends ContextResult {
+}
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/removeAllValue")
+class ContextResetIDResult extends ContextResult {
+}
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/removeValue")
+class ContextRemoveValueResult extends ContextResult {
+}
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/onBindIDListener")
+class ContextBindIDResult extends ContextResult {
+}
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/onBindKeyListener")
+class ContextBindKeyResult extends ContextResult {
+}
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/searchContextValue")
+class ContextSearchResult extends ContextResult {
+}
+
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/getContextValue")
+class ContextGetValueResult extends ContextResult {
+}
+
+// TODO: 用来匹配所有的void 不需要处理返回值的result
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/(createHistory|removeHistory|removeAllValueByKeyPrefix|removeAllValueByKeyPrefixAndContextType)")
+class VoidResult extends ContextResult {
+}
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/(getHistories|searchHistory)")
+class ContextHistoriesGetResult extends ContextResult {
+ var contextHistory:java.util.List[String] = _
+ def getContextHistory:java.util.List[String] = this.contextHistory
+ def setContextHistory(contextHistory:java.util.List[String])= this.contextHistory = contextHistory
+}
+
+@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/getHistory")
+class ContextHistoryGetResult extends ContextResult {
+ var contextHistory:String = _
+ def getContextHistory:String = this.contextHistory
+ def setContextHistory(contextHistory:String)= this.contextHistory = contextHistory
+}
+
+
diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientConf.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientConf.scala
new file mode 100644
index 0000000..2fc00ae
--- /dev/null
+++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientConf.scala
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.utils
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars
+
+/**
+ * created by cooperyang on 2020/2/18
+ * Description:
+ */
+object ContextClientConf {
+ val LINKIS_WEB_VERSION:CommonVars[String] = CommonVars[String]("wds.linkis.web.version", "v1")
+
+
+ val CONTEXT_CLIENT_AUTH_KEY:CommonVars[String] = CommonVars[String]("wds.linkis.context.client.auth.key", "Token-Code")
+
+ val CONTEXT_CLIENT_AUTH_VALUE:CommonVars[String] = CommonVars[String]("wds.linkis.context.client.auth.value", "BML-AUTH")
+
+
+ val URL_PREFIX:CommonVars[String] = CommonVars[String]("wds.linkis.cs.url.prefix", "/api/rest_j/v1/contextservice", "cs服务的url前缀")
+
+ val CREATE_CONTEXT_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.createcontext.url", "createContextID")
+
+ val SET_VALUE_BY_KEY_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.setvaluebykey.url", "setValueByKey")
+
+ val SET_KEYVALUE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.setkeyvalue.url", "setValue")
+
+ val RESET_KEYVALUE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.resetkeyvalue.url", "resetValue")
+
+ val REMOVE_VALUE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.removeValue.url", "removeValue")
+
+ val RESET_CONTEXT_ID_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.reset.contextid.url", "removeAllValue")
+
+ val HEART_BEAT_ENABLED:CommonVars[String] = CommonVars[String]("wds.linkis.cs.heartbeat.enabled", "true")
+
+}
diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientUtils.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientUtils.scala
new file mode 100644
index 0000000..f0b8953
--- /dev/null
+++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientUtils.scala
@@ -0,0 +1,33 @@
+/**
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.utils
+
+import java.lang
+import java.lang.reflect.Type
+
+import com.google.gson.{GsonBuilder, JsonElement, JsonPrimitive, JsonSerializationContext, JsonSerializer}
+
+/**
+ * created by cooperyang on 2020/2/23
+ * Description:
+ */
+object ContextClientUtils {
+ implicit val gson = new GsonBuilder().setPrettyPrinting().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls
+ .registerTypeAdapter(classOf[java.lang.Double], new JsonSerializer[java.lang.Double] {
+ override def serialize(t: lang.Double, `type`: Type, jsonSerializationContext: JsonSerializationContext): JsonElement =
+ if(t == t.longValue()) new JsonPrimitive(t.longValue()) else new JsonPrimitive(t)
+ }).create
+}
diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextServerHttpConf.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextServerHttpConf.scala
new file mode 100644
index 0000000..474405d
--- /dev/null
+++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextServerHttpConf.scala
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.client.utils
+
+import com.webank.wedatasphere.linkis.common.conf.Configuration
+
+/**
+ * created by cooperyang on 2020/2/19
+ * Description:
+ */
+object ContextServerHttpConf {
+ val gatewayInstance: String = Configuration.getGateWayURL()
+ val urlPrefix: String = if (ContextClientConf.URL_PREFIX.getValue.endsWith("/")) {
+ ContextClientConf.URL_PREFIX.getValue.substring(0, ContextClientConf.URL_PREFIX.getValue.length - 1)
+ } else ContextClientConf.URL_PREFIX.getValue
+
+ val createContextURL:String = urlPrefix + "/" + ContextClientConf.CREATE_CONTEXT_URL.getValue
+
+ val updateContextURL:String = urlPrefix + "/" + ContextClientConf.SET_VALUE_BY_KEY_URL.getValue;
+
+ val setKeyValueURL:String = urlPrefix + "/" + ContextClientConf.SET_KEYVALUE_URL.getValue
+
+ val resetKeyValueURL:String = urlPrefix + "/" + ContextClientConf.RESET_KEYVALUE_URL.getValue
+
+ val removeValueURL:String = urlPrefix + "/" + ContextClientConf.REMOVE_VALUE_URL.getValue
+
+ val resetContextIdURL:String = urlPrefix + "/" + ContextClientConf.RESET_CONTEXT_ID_URL.getValue
+
+ val onBindKeyURL:String = urlPrefix + "/" + "onBindKeyListener"
+
+ val onBindIDURL:String = urlPrefix + "/" + "onBindIDListener"
+
+ val getContextIDURL:String = urlPrefix + "/" + "getContextID"
+
+ val heartBeatURL:String = urlPrefix + "/" + "heartbeat"
+
+ val searchURL:String = urlPrefix + "/" + "searchContextValue"
+
+ val getContextValueURL:String = urlPrefix + "/" + "getContextValue"
+
+ val createContextHistory:String = urlPrefix + "/" + "createHistory"
+
+ val removeContextHistory:String = urlPrefix + "/" + "removeHistory"
+
+ val getContextHistories:String = urlPrefix + "/" + "getHistories"
+
+ val getContextHistory:String = urlPrefix + "/" + "getHistory"
+
+ val searchContextHistory:String = urlPrefix + "/" + "searchHistory"
+
+ val removeAllValueByKeyPrefixAndContextTypeURL: String = urlPrefix + "/" + "removeAllValueByKeyPrefixAndContextType"
+
+ val removeAllValueByKeyPrefixURL: String = urlPrefix + "/" + "removeAllValueByKeyPrefix"
+}
diff --git a/contextservice/cs-common/pom.xml b/contextservice/cs-common/pom.xml
new file mode 100644
index 0000000..6cde58b
--- /dev/null
+++ b/contextservice/cs-common/pom.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-common</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-common</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.10</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/annotation/KeywordMethod.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/annotation/KeywordMethod.java
new file mode 100644
index 0000000..dcf96b7
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/annotation/KeywordMethod.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.annotation;
+
+import java.lang.annotation.*;
+
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 16:15
+ */
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+@Documented
+public @interface KeywordMethod {
+
+ String splitter() default "";
+
+ String regex() default "";
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/CSResultData.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/CSResultData.java
new file mode 100644
index 0000000..bba746e
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/CSResultData.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.data;
+
+import com.webank.wedatasphere.linkis.common.io.FsPath;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 19:20
+ */
+public class CSResultData implements Data {
+
+ private FsPath fsPath;
+
+ @Override
+ public FsPath getLocation() {
+ return null;
+ }
+
+ @Override
+ public void setLocation(FsPath fsPath) {
+
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/Data.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/Data.java
new file mode 100644
index 0000000..0b5dd0d
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/Data.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.data;
+
+import com.webank.wedatasphere.linkis.common.io.FsPath;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public interface Data {
+
+ FsPath getLocation();
+
+ void setLocation(FsPath fsPath);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/JobData.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/JobData.java
new file mode 100644
index 0000000..5ffa406
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/JobData.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.data;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 22:07
+ */
+public interface JobData {
+
+ long getJobID();
+
+ void setJobID(long jobID);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/LinkisJobData.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/LinkisJobData.java
new file mode 100644
index 0000000..fe430d3
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/data/LinkisJobData.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.data;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 22:08
+ */
+public class LinkisJobData implements JobData {
+
+ private long jobID;
+
+ @Override
+ public long getJobID() {
+ return this.jobID;
+ }
+
+ @Override
+ public void setJobID(long jobID) {
+ this.jobID = jobID;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ContextScope.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ContextScope.java
new file mode 100644
index 0000000..135a9e0
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ContextScope.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.enumeration;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public enum ContextScope {
+ /**
+ *
+ */
+ PRIVATE,PUBLIC,PROTECTED,FRIENDLY
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ContextType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ContextType.java
new file mode 100644
index 0000000..fd65946
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ContextType.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.enumeration;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public enum ContextType {
+ /**
+ *
+ */
+ METADATA,DATA,RESOURCE,OBJECT,ENV,COST,UDF,Variable
+}
+
+
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/DBType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/DBType.java
new file mode 100644
index 0000000..f64aed5
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/DBType.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.enumeration;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public enum DBType {
+ /**
+ *
+ */
+ CS,HIVE,MIDE,ES,HABSE
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ExpireType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ExpireType.java
new file mode 100644
index 0000000..94dc29f
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/ExpireType.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.enumeration;
+
+/**
+ * Created by patinousward on 2020/2/12.
+ */
+public enum ExpireType {
+ /**
+ *
+ */
+ TODAY(0), MAX_EXISTS_TIME(1), NEVER(2);
+
+ private Integer id;
+
+ ExpireType(Integer id) {
+ this.id = id;
+ }
+
+ public Integer getId() {
+ return this.id;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/WorkType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/WorkType.java
new file mode 100644
index 0000000..f8e4c23
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/enumeration/WorkType.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.enumeration;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/21
+ */
+public enum WorkType {
+ WORKSPACE,
+ PROJECT,
+ FLOW,
+ NODE
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/Env.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/Env.java
new file mode 100644
index 0000000..a4ff7cb
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/Env.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.env;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface Env {
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/OSEnv.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/OSEnv.java
new file mode 100644
index 0000000..a5c86bd
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/OSEnv.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.env;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface OSEnv extends Env {
+
+ String getOs();
+
+ void setOs(String os);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/SoftWareEnv.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/SoftWareEnv.java
new file mode 100644
index 0000000..b06de4e
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/SoftWareEnv.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.env;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public interface SoftWareEnv {
+
+ String getSoftware();
+
+ void setSoftware(String software);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/SoftwareLibEnv.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/SoftwareLibEnv.java
new file mode 100644
index 0000000..5f66cf6
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/env/SoftwareLibEnv.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.env;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public interface SoftwareLibEnv extends Env {
+
+ String getLib();
+
+ void setLib(String lib);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/CommonContextIDListenerDomain.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/CommonContextIDListenerDomain.java
new file mode 100644
index 0000000..ec73fa4
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/CommonContextIDListenerDomain.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.listener;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * Created by patinousward on 2020/2/28.
+ */
+public class CommonContextIDListenerDomain implements ContextIDListenerDomain {
+
+ private String source;
+
+ private ContextID contextID;
+
+ @Override
+ public String getSource() {
+ return this.source;
+ }
+
+ @Override
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ @Override
+ public ContextID getContextID() {
+ return this.contextID;
+ }
+
+ @Override
+ public void setContextID(ContextID contextID) {
+ this.contextID = contextID;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/CommonContextKeyListenerDomain.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/CommonContextKeyListenerDomain.java
new file mode 100644
index 0000000..4cc2ad4
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/CommonContextKeyListenerDomain.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.listener;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+
+/**
+ * Created by patinousward on 2020/2/28.
+ */
+public class CommonContextKeyListenerDomain implements ContextKeyListenerDomain {
+
+ private String source;
+
+ private ContextKey contextKey;
+
+ private ContextID contextID;
+
+ public ContextID getContextID() {
+ return contextID;
+ }
+
+ public void setContextID(ContextID contextID) {
+ this.contextID = contextID;
+ }
+
+
+
+ @Override
+ public String getSource() {
+ return this.source;
+ }
+
+ @Override
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ @Override
+ public ContextKey getContextKey() {
+ return this.contextKey;
+ }
+
+ @Override
+ public void setContextKey(ContextKey contextKey) {
+ this.contextKey = contextKey;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ContextIDListenerDomain.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ContextIDListenerDomain.java
new file mode 100644
index 0000000..5c9c761
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ContextIDListenerDomain.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.listener;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextIDListenerDomain extends ListenerDomain {
+
+ ContextID getContextID();
+
+ void setContextID(ContextID contextID);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ContextKeyListenerDomain.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ContextKeyListenerDomain.java
new file mode 100644
index 0000000..e87ba82
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ContextKeyListenerDomain.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.listener;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextKeyListenerDomain extends ListenerDomain {
+
+ ContextKey getContextKey();
+
+ void setContextKey(ContextKey contextKey);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ListenerDomain.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ListenerDomain.java
new file mode 100644
index 0000000..054b09c
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/listener/ListenerDomain.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.listener;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ListenerDomain {
+
+ String getSource();
+
+ void setSource(String source);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSColumn.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSColumn.java
new file mode 100644
index 0000000..469a17f
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSColumn.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public class CSColumn implements Column {
+
+ private String name;
+ private String alias;
+ private String type;
+ private String comment;
+ private String express;
+ private String rule;
+ private Boolean isPrimary;
+ private Integer length;
+
+ @Override
+ public Integer getLength() {
+ return length;
+ }
+
+ @Override
+ public void setLength(Integer length) {
+ this.length = length;
+ }
+
+ @Override
+ @KeywordMethod
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getAlias() {
+ return alias;
+ }
+
+ @Override
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ @Override
+ public String getType() {
+ return type;
+ }
+
+ @Override
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public String getComment() {
+ return comment;
+ }
+
+ @Override
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String getExpress() {
+ return express;
+ }
+
+ @Override
+ public void setExpress(String express) {
+ this.express = express;
+ }
+
+ @Override
+ public String getRule() {
+ return rule;
+ }
+
+ @Override
+ public void setRule(String rule) {
+ this.rule = rule;
+ }
+
+ @Override
+ public Boolean getPrimary() {
+ return isPrimary;
+ }
+
+ @Override
+ public void setPrimary(Boolean primary) {
+ isPrimary = primary;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSDB.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSDB.java
new file mode 100644
index 0000000..d79b063
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSDB.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.DBType;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public class CSDB implements DB {
+
+ private String name;
+
+ private DBType dbType;
+
+ private String[] lables;
+
+ private String comment;
+
+ private String owners;
+
+ public static DB build() {
+ return null;
+ }
+
+ ;
+
+ @Override
+ @KeywordMethod
+ public String getName() {
+ return this.name;
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public DBType getDbType() {
+ return this.dbType;
+ }
+
+ @Override
+ public void setDbType(DBType dbType) {
+ this.dbType = dbType;
+ }
+
+ @Override
+ @KeywordMethod
+ public String getOwners() {
+ return this.owners;
+ }
+
+ @Override
+ public void setOwners(String owners) {
+ this.owners = owners;
+ }
+
+ @Override
+ public String getComment() {
+ return this.comment;
+ }
+
+ @Override
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String[] getLables() {
+ return this.lables;
+ }
+
+ @Override
+ public void setLables(String[] lables) {
+ this.lables = lables;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSPartition.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSPartition.java
new file mode 100644
index 0000000..6bc0cfc
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSPartition.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/3/4.
+ */
+public class CSPartition implements Partition {
+
+ private String name;
+ private String alias;
+ private String type;
+ private String comment;
+ private String express;
+ private String rule;
+ private Boolean isPrimary;
+ private Integer length;
+ private List<String> value;
+
+
+ @Override
+ public Integer getLength() {
+ return length;
+ }
+
+ @Override
+ public void setLength(Integer length) {
+ this.length = length;
+ }
+
+ @Override
+ @KeywordMethod
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getAlias() {
+ return alias;
+ }
+
+ @Override
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ @Override
+ public String getType() {
+ return type;
+ }
+
+ @Override
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ @Override
+ public String getComment() {
+ return comment;
+ }
+
+ @Override
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String getExpress() {
+ return express;
+ }
+
+ @Override
+ public void setExpress(String express) {
+ this.express = express;
+ }
+
+ @Override
+ public String getRule() {
+ return rule;
+ }
+
+ @Override
+ public void setRule(String rule) {
+ this.rule = rule;
+ }
+
+ @Override
+ public Boolean getPrimary() {
+ return isPrimary;
+ }
+
+ @Override
+ public void setPrimary(Boolean primary) {
+ isPrimary = primary;
+ }
+
+ public List<String> getValue() {
+ return value;
+ }
+
+ public void setValue(List<String> value) {
+ this.value = value;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSTable.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSTable.java
new file mode 100644
index 0000000..876e163
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/CSTable.java
@@ -0,0 +1,254 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public class CSTable implements Table {
+
+ private String name;
+ private String alias;
+ private String creator;
+ private String comment;
+ private Date createTime;
+ private String productName;
+ private String projectName;
+ private String usage;
+ private Integer lifecycle;
+ private Integer useWay;
+ private Boolean isImport;
+ private Integer modelLevel;
+ private Boolean isExternalUse;
+ private Boolean isPartitionTable;
+ private Boolean isAvailable;
+ private Boolean isView;
+ private String location;
+ private CSColumn[] columns;
+ private List<CSPartition> partitions;
+ private CSDB db;
+
+ @Override
+ @KeywordMethod
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getAlias() {
+ return alias;
+ }
+
+ @KeywordMethod
+ public String getLocation() {
+ return location;
+ }
+
+ public void setLocation(String location) {
+ this.location = location;
+ }
+
+ @Override
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ @Override
+ public String getCreator() {
+ return creator;
+ }
+
+ @Override
+ public void setCreator(String creator) {
+ this.creator = creator;
+ }
+
+ @Override
+ public String getComment() {
+ return comment;
+ }
+
+ @Override
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public Date getCreateTime() {
+ return createTime;
+ }
+
+ @Override
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ @Override
+ public String getProductName() {
+ return productName;
+ }
+
+ @Override
+ public void setProductName(String productName) {
+ this.productName = productName;
+ }
+
+ @Override
+ public String getProjectName() {
+ return projectName;
+ }
+
+ @Override
+ public void setProjectName(String projectName) {
+ this.projectName = projectName;
+ }
+
+ @Override
+ public String getUsage() {
+ return usage;
+ }
+
+ @Override
+ public void setUsage(String usage) {
+ this.usage = usage;
+ }
+
+ @Override
+ public Integer getLifecycle() {
+ return lifecycle;
+ }
+
+ @Override
+ public void setLifecycle(Integer lifecycle) {
+ this.lifecycle = lifecycle;
+ }
+
+ @Override
+ public Integer getUseWay() {
+ return useWay;
+ }
+
+ @Override
+ public void setUseWay(Integer useWay) {
+ this.useWay = useWay;
+ }
+
+ @Override
+ public Boolean getImport() {
+ return isImport;
+ }
+
+ @Override
+ public void setImport(Boolean anImport) {
+ isImport = anImport;
+ }
+
+ @Override
+ public Integer getModelLevel() {
+ return modelLevel;
+ }
+
+ @Override
+ public void setModelLevel(Integer modelLevel) {
+ this.modelLevel = modelLevel;
+ }
+
+ @Override
+ public Boolean getExternalUse() {
+ return isExternalUse;
+ }
+
+ @Override
+ public void setExternalUse(Boolean externalUse) {
+ isExternalUse = externalUse;
+ }
+
+ @Override
+ public Boolean getPartitionTable() {
+ return isPartitionTable;
+ }
+
+ @Override
+ public void setPartitionTable(Boolean partitionTable) {
+ isPartitionTable = partitionTable;
+ }
+
+ @Override
+ public Boolean getAvailable() {
+ return isAvailable;
+ }
+
+ @Override
+ public void setAvailable(Boolean available) {
+ isAvailable = available;
+ }
+
+ public Boolean isView() {
+ return this.isView;
+ }
+
+ @Override
+ public Boolean getView() {
+ return isView;
+ }
+
+ @Override
+ public void setView(Boolean view) {
+ isView = view;
+ }
+
+ @Override
+ public CSColumn[] getColumns() {
+ return this.columns;
+ }
+
+ @Override
+ public void setColumns(CSColumn[] columns) {
+ this.columns = columns;
+ }
+
+
+ @Override
+ public List<CSPartition> getPartitions() {
+ return partitions;
+ }
+
+ @Override
+ public void setPartitions(List<CSPartition> partitions) {
+ this.partitions = partitions;
+ }
+
+ @Override
+ public CSDB getDb() {
+ return db;
+ }
+
+ @Override
+ public void setDb(CSDB db) {
+ this.db = db;
+ }
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Column.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Column.java
new file mode 100644
index 0000000..902ef0c
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Column.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface Column {
+
+ Integer getLength();
+
+ void setLength(Integer length);
+
+ String getName();
+
+ void setName(String name);
+
+ String getAlias();
+
+ void setAlias(String alias);
+
+ String getType();
+
+ void setType(String type);
+
+ String getComment();
+
+ void setComment(String comment);
+
+ String getExpress();
+
+ void setExpress(String express);
+
+ String getRule();
+
+ void setRule(String rule);
+
+ Boolean getPrimary();
+
+ void setPrimary(Boolean primary);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/DB.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/DB.java
new file mode 100644
index 0000000..3e468f2
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/DB.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.DBType;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface DB {
+
+ String getName();
+
+ void setName(String name);
+
+ DBType getDbType();
+
+ void setDbType(DBType dbType);
+
+ String getOwners();
+
+ void setOwners(String owners);
+
+ String getComment();
+
+ void setComment(String comment);
+
+ String[] getLables();
+
+ void setLables(String[] lables);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Partition.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Partition.java
new file mode 100644
index 0000000..4790595
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Partition.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface Partition {
+
+ Integer getLength();
+
+ void setLength(Integer length);
+
+ String getName();
+
+ void setName(String name);
+
+ String getAlias();
+
+ void setAlias(String alias);
+
+ String getType();
+
+ void setType(String type);
+
+ String getComment();
+
+ void setComment(String comment);
+
+ String getExpress();
+
+ void setExpress(String express);
+
+ String getRule();
+
+ void setRule(String rule);
+
+ Boolean getPrimary();
+
+ void setPrimary(Boolean primary);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Table.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Table.java
new file mode 100644
index 0000000..69e1efb
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/metadata/Table.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.metadata;
+
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface Table {
+
+ String getName();
+
+ void setName(String name);
+
+ String getAlias();
+
+ void setAlias(String alias);
+
+ String getCreator();
+
+ void setCreator(String creator);
+
+ String getComment();
+
+ void setComment(String comment);
+
+ Date getCreateTime();
+
+ void setCreateTime(Date createTime);
+
+ String getProductName();
+
+ void setProductName(String productName);
+
+ String getProjectName();
+
+ void setProjectName(String projectName);
+
+ String getUsage();
+
+ void setUsage(String usage);
+
+ Integer getLifecycle();
+
+ void setLifecycle(Integer lifecycle);
+
+ Integer getUseWay();
+
+ void setUseWay(Integer useWay);
+
+ Boolean getImport();
+
+ void setImport(Boolean anImport);
+
+ Integer getModelLevel();
+
+ void setModelLevel(Integer modelLevel);
+
+ Boolean getExternalUse();
+
+ void setExternalUse(Boolean externalUse);
+
+ Boolean getPartitionTable();
+
+ void setPartitionTable(Boolean partitionTable);
+
+ Boolean getAvailable();
+
+ void setAvailable(Boolean available);
+
+ Boolean getView();
+
+ CSDB getDb();
+
+ void setDb(CSDB db);
+
+ void setView(Boolean view);
+
+ CSColumn[] getColumns();
+
+ void setColumns(CSColumn[] columns);
+
+ List<CSPartition> getPartitions();
+
+ void setPartitions(List<CSPartition> partitions);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSFlowInfos.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSFlowInfos.java
new file mode 100644
index 0000000..4471c7a
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSFlowInfos.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.object;
+
+import java.util.Map;
+
+/**
+ * @author peacewong
+ * @date 2020/3/10 11:36
+ */
+public class CSFlowInfos implements CSInfos {
+
+ private Map<String, Object> infos;
+
+
+ @Override
+ public Map<String, Object> getInfos() {
+ return this.infos;
+ }
+
+ @Override
+ public void setInfos(Map<String, Object> infos) {
+ this.infos = infos;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSInfos.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSInfos.java
new file mode 100644
index 0000000..a3afdbc
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSInfos.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.object;
+
+import java.util.Map;
+
+/**
+ * @author peacewong
+ * @date 2020/3/10 11:35
+ */
+public interface CSInfos {
+
+ Map<String, Object> getInfos();
+
+ void setInfos(Map<String, Object> infos);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSProperty.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSProperty.java
new file mode 100644
index 0000000..80bef85
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CSProperty.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.object;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface CSProperty {
+
+ String getKey();
+
+ void setKey(String key);
+
+ String getValue();
+
+ void setValue(String value);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CommonGlobalConfiguration.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CommonGlobalConfiguration.java
new file mode 100644
index 0000000..b37f6ae
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CommonGlobalConfiguration.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.object;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public class CommonGlobalConfiguration extends CommonProperty {
+
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CommonProperty.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CommonProperty.java
new file mode 100644
index 0000000..635dabd
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/CommonProperty.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.object;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public class CommonProperty implements CSProperty {
+
+ private String key;
+
+ private String value;
+
+
+ @Override
+ @KeywordMethod
+ public String getKey() {
+ return this.key;
+ }
+
+ @Override
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String getValue() {
+ return this.value;
+ }
+
+ @Override
+ public void setValue(String value) {
+ this.value = value;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/LinkisVariable.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/LinkisVariable.java
new file mode 100644
index 0000000..793af76
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/object/LinkisVariable.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.object;
+
+/**
+ * @author peacewong
+ * @date 2020/3/4 20:02
+ */
+public class LinkisVariable extends CommonProperty {
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/BMLResource.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/BMLResource.java
new file mode 100644
index 0000000..9626c5e
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/BMLResource.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.resource;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+import java.util.Date;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface BMLResource extends Resource {
+
+ @KeywordMethod
+ String getResourceId();
+
+ String getVersion();
+
+ String getUser();
+
+ void setUser(String user);
+
+ String getSystem();
+
+ void setSystem(String system);
+
+ Boolean getEnableFlag();
+
+ void setEnableFlag(Boolean enableFlag);
+
+ Boolean getPrivate();
+
+ void setPrivate(Boolean aPrivate);
+
+ String getResourceHeader();
+
+ void setResourceHeader(String resourceHeader);
+
+ @KeywordMethod
+ String getDownloadedFileName();
+
+ void setDownloadedFileName(String downloadedFileName);
+
+ String getSys();
+
+ void setSys(String sys);
+
+ Date getCreateTime();
+
+ void setCreateTime(Date createTime);
+
+ Boolean getExpire();
+
+ void setExpire(Boolean expire);
+
+ String getExpireType();
+
+ void setExpireType(String expireType);
+
+ String getExpireTime();
+
+ void setExpireTime(String expireTime);
+
+ Date getUpdateTime();
+
+ void setUpdateTime(Date updateTime);
+
+ String getUpdator();
+
+ void setUpdator(String updator);
+
+ Integer getMaxVersion();
+
+ void setMaxVersion(Integer maxVersion);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/CommonFxResource.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/CommonFxResource.java
new file mode 100644
index 0000000..b3545f1
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/CommonFxResource.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.resource;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+import java.util.Date;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public class CommonFxResource implements FxResource {
+
+ private String createUser;
+ private String udfName;
+ private Integer udfType;
+ private String path;
+ private String registerFormat;
+ private String useFormat;
+ private String description;
+ private Boolean isExpire;
+ private Boolean isShared;
+ private Long treeId;
+ private Date createTime;
+ private Date updateTime;
+ private Boolean isLoad;
+
+ @Override
+ @KeywordMethod
+ public String getCreateUser() {
+ return createUser;
+ }
+
+ @Override
+ public void setCreateUser(String createUser) {
+ this.createUser = createUser;
+ }
+
+ @Override
+ @KeywordMethod
+ public String getUdfName() {
+ return udfName;
+ }
+
+ @Override
+ public void setUdfName(String udfName) {
+ this.udfName = udfName;
+ }
+
+ @Override
+ @KeywordMethod
+ public Integer getUdfType() {
+ return udfType;
+ }
+
+ @Override
+ public void setUdfType(Integer udfType) {
+ this.udfType = udfType;
+ }
+
+ @Override
+ public String getPath() {
+ return path;
+ }
+
+ @Override
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ @Override
+ public String getRegisterFormat() {
+ return registerFormat;
+ }
+
+ @Override
+ public void setRegisterFormat(String registerFormat) {
+ this.registerFormat = registerFormat;
+ }
+
+ @Override
+ public String getUseFormat() {
+ return useFormat;
+ }
+
+ @Override
+ public void setUseFormat(String useFormat) {
+ this.useFormat = useFormat;
+ }
+
+ @Override
+ public String getDescription() {
+ return description;
+ }
+
+ @Override
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ @Override
+ public Boolean getExpire() {
+ return isExpire;
+ }
+
+ @Override
+ public void setExpire(Boolean expire) {
+ isExpire = expire;
+ }
+
+ @Override
+ public Boolean getShared() {
+ return isShared;
+ }
+
+ @Override
+ public void setShared(Boolean shared) {
+ isShared = shared;
+ }
+
+ @Override
+ public Long getTreeId() {
+ return treeId;
+ }
+
+ @Override
+ public void setTreeId(Long treeId) {
+ this.treeId = treeId;
+ }
+
+ @Override
+ public Date getCreateTime() {
+ return createTime;
+ }
+
+ @Override
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ @Override
+ public Date getUpdateTime() {
+ return updateTime;
+ }
+
+ @Override
+ public void setUpdateTime(Date updateTime) {
+ this.updateTime = updateTime;
+ }
+
+ @Override
+ public Boolean getLoad() {
+ return isLoad;
+ }
+
+ @Override
+ public void setLoad(Boolean load) {
+ isLoad = load;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/CommonUDFResource.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/CommonUDFResource.java
new file mode 100644
index 0000000..bc4394a
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/CommonUDFResource.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.resource;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+import java.util.Date;
+
+/**
+ * Created by patinousward on 2020/3/3.
+ */
+public class CommonUDFResource implements FxResource {
+
+ private String createUser;
+ private String udfName;
+ private Integer udfType;
+ private String path;
+ private String registerFormat;
+ private String useFormat;
+ private String description;
+ private Boolean isExpire;
+ private Boolean isShared;
+ private Long treeId;
+ private Date createTime;
+ private Date updateTime;
+ private Boolean isLoad;
+
+ @Override
+ @KeywordMethod
+ public String getCreateUser() {
+ return createUser;
+ }
+
+ @Override
+ public void setCreateUser(String createUser) {
+ this.createUser = createUser;
+ }
+
+ @Override
+ @KeywordMethod
+ public String getUdfName() {
+ return udfName;
+ }
+
+ @Override
+ public void setUdfName(String udfName) {
+ this.udfName = udfName;
+ }
+
+ @Override
+ @KeywordMethod
+ public Integer getUdfType() {
+ return udfType;
+ }
+
+ @Override
+ public void setUdfType(Integer udfType) {
+ this.udfType = udfType;
+ }
+
+ @Override
+ public String getPath() {
+ return path;
+ }
+
+ @Override
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ @Override
+ public String getRegisterFormat() {
+ return registerFormat;
+ }
+
+ @Override
+ public void setRegisterFormat(String registerFormat) {
+ this.registerFormat = registerFormat;
+ }
+
+ @Override
+ public String getUseFormat() {
+ return useFormat;
+ }
+
+ @Override
+ public void setUseFormat(String useFormat) {
+ this.useFormat = useFormat;
+ }
+
+ @Override
+ public String getDescription() {
+ return description;
+ }
+
+ @Override
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ @Override
+ public Boolean getExpire() {
+ return isExpire;
+ }
+
+ @Override
+ public void setExpire(Boolean expire) {
+ isExpire = expire;
+ }
+
+ @Override
+ public Boolean getShared() {
+ return isShared;
+ }
+
+ @Override
+ public void setShared(Boolean shared) {
+ isShared = shared;
+ }
+
+ @Override
+ public Long getTreeId() {
+ return treeId;
+ }
+
+ @Override
+ public void setTreeId(Long treeId) {
+ this.treeId = treeId;
+ }
+
+ @Override
+ public Date getCreateTime() {
+ return createTime;
+ }
+
+ @Override
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ @Override
+ public Date getUpdateTime() {
+ return updateTime;
+ }
+
+ @Override
+ public void setUpdateTime(Date updateTime) {
+ this.updateTime = updateTime;
+ }
+
+ @Override
+ public Boolean getLoad() {
+ return isLoad;
+ }
+
+ @Override
+ public void setLoad(Boolean load) {
+ isLoad = load;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/FxResource.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/FxResource.java
new file mode 100644
index 0000000..0c26f7f
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/FxResource.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.resource;
+
+import java.util.Date;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface FxResource extends Resource {
+
+ String getCreateUser();
+
+ void setCreateUser(String createUser);
+
+ String getUdfName();
+
+ void setUdfName(String udfName);
+
+ Integer getUdfType();
+
+ void setUdfType(Integer udfType);
+
+ String getPath();
+
+ void setPath(String path);
+
+ String getRegisterFormat();
+
+ void setRegisterFormat(String registerFormat);
+
+ String getUseFormat();
+
+ void setUseFormat(String useFormat);
+
+ String getDescription();
+
+ void setDescription(String description);
+
+ Boolean getExpire();
+
+ void setExpire(Boolean expire);
+
+ Boolean getShared();
+
+ void setShared(Boolean shared);
+
+ Long getTreeId();
+
+ void setTreeId(Long treeId);
+
+ Date getCreateTime();
+
+ void setCreateTime(Date createTime);
+
+ Date getUpdateTime();
+
+ void setUpdateTime(Date updateTime);
+
+ Boolean getLoad();
+
+ void setLoad(Boolean load);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/LinkisBMLResource.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/LinkisBMLResource.java
new file mode 100644
index 0000000..7431c65
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/LinkisBMLResource.java
@@ -0,0 +1,220 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.resource;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+
+import java.util.Date;
+
+/**
+ * created by cooperyang on 2020/2/27
+ * Description:
+ */
+public class LinkisBMLResource implements BMLResource {
+
+
+ private String resourceId;
+
+ private String version;
+
+ private Boolean isPrivate;
+
+ private String resourceHeader;
+
+ private String downloadedFileName;
+
+ private String sys;
+
+ private Date createTime;
+
+ private Boolean isExpire;
+
+ private String expireType;
+
+ private String expireTime;
+
+ private Date updateTime;
+
+ private String updator;
+
+ private Integer maxVersion;
+
+ private String user;
+
+ private String system;
+
+ private Boolean enableFlag;
+
+ @Override
+ public Boolean getPrivate() {
+ return isPrivate;
+ }
+
+ @Override
+ public void setPrivate(Boolean aPrivate) {
+ isPrivate = aPrivate;
+ }
+
+ @Override
+ public String getResourceHeader() {
+ return resourceHeader;
+ }
+
+ @Override
+ public void setResourceHeader(String resourceHeader) {
+ this.resourceHeader = resourceHeader;
+ }
+
+ @Override
+ public String getDownloadedFileName() {
+ return downloadedFileName;
+ }
+
+ @Override
+ public void setDownloadedFileName(String downloadedFileName) {
+ this.downloadedFileName = downloadedFileName;
+ }
+
+ @Override
+ public String getSys() {
+ return sys;
+ }
+
+ @Override
+ public void setSys(String sys) {
+ this.sys = sys;
+ }
+
+ @Override
+ public Date getCreateTime() {
+ return createTime;
+ }
+
+ @Override
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ @Override
+ public Boolean getExpire() {
+ return isExpire;
+ }
+
+ @Override
+ public void setExpire(Boolean expire) {
+ isExpire = expire;
+ }
+
+ @Override
+ public String getExpireType() {
+ return expireType;
+ }
+
+ @Override
+ public void setExpireType(String expireType) {
+ this.expireType = expireType;
+ }
+
+ @Override
+ public String getExpireTime() {
+ return expireTime;
+ }
+
+ @Override
+ public void setExpireTime(String expireTime) {
+ this.expireTime = expireTime;
+ }
+
+ @Override
+ public Date getUpdateTime() {
+ return updateTime;
+ }
+
+ @Override
+ public void setUpdateTime(Date updateTime) {
+ this.updateTime = updateTime;
+ }
+
+ @Override
+ public String getUpdator() {
+ return updator;
+ }
+
+ @Override
+ public void setUpdator(String updator) {
+ this.updator = updator;
+ }
+
+ @Override
+ public Integer getMaxVersion() {
+ return maxVersion;
+ }
+
+ @Override
+ public void setMaxVersion(Integer maxVersion) {
+ this.maxVersion = maxVersion;
+ }
+
+ @Override
+ public String getUser() {
+ return user;
+ }
+
+ @Override
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ @Override
+ public String getSystem() {
+ return system;
+ }
+
+ @Override
+ public void setSystem(String system) {
+ this.system = system;
+ }
+
+ @Override
+ public Boolean getEnableFlag() {
+ return enableFlag;
+ }
+
+ @Override
+ public void setEnableFlag(Boolean enableFlag) {
+ this.enableFlag = enableFlag;
+ }
+
+ @Override
+ @KeywordMethod
+ public String getResourceId() {
+ return this.resourceId;
+ }
+
+ @Override
+ @KeywordMethod
+ public String getVersion() {
+ return this.version;
+ }
+
+ public void setResourceId(String resourceId) {
+ this.resourceId = resourceId;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/Resource.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/Resource.java
new file mode 100644
index 0000000..4259d62
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/Resource.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.resource;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ValueBean;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface Resource {
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/UDFResource.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/UDFResource.java
new file mode 100644
index 0000000..e8e89bd
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/resource/UDFResource.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.resource;
+
+import java.util.Date;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface UDFResource extends Resource {
+
+ String getCreateUser();
+
+ void setCreateUser(String createUser);
+
+ String getUdfName();
+
+ void setUdfName(String udfName);
+
+ Integer getUdfType();
+
+ void setUdfType(Integer udfType);
+
+ String getPath();
+
+ void setPath(String path);
+
+ String getRegisterFormat();
+
+ void setRegisterFormat(String registerFormat);
+
+ String getUseFormat();
+
+ void setUseFormat(String useFormat);
+
+ String getDescription();
+
+ void setDescription(String description);
+
+ Boolean getExpire();
+
+ void setExpire(Boolean expire);
+
+ Boolean getShared();
+
+ void setShared(Boolean shared);
+
+ Long getTreeId();
+
+ void setTreeId(Long treeId);
+
+ Date getCreateTime();
+
+ void setCreateTime(Date createTime);
+
+ Date getUpdateTime();
+
+ void setUpdateTime(Date updateTime);
+
+ Boolean getLoad();
+
+ void setLoad(Boolean load);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CombinedNodeIDContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CombinedNodeIDContextID.java
new file mode 100644
index 0000000..69a5810
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CombinedNodeIDContextID.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * @author peacewong
+ * @date 2020/3/10 23:18
+ */
+public class CombinedNodeIDContextID extends LinkisHAWorkFlowContextID {
+
+ public CombinedNodeIDContextID() {
+
+ }
+
+ public CombinedNodeIDContextID(ContextID contextID, String nodeID) {
+
+ this.setContextId(contextID.getContextId());
+ this.nodeID = nodeID;
+ if (contextID instanceof LinkisHAWorkFlowContextID) {
+ LinkisHAWorkFlowContextID haWorkFlowContextID = (LinkisHAWorkFlowContextID) contextID;
+ setBackupInstance(haWorkFlowContextID.getBackupInstance());
+ setInstance(haWorkFlowContextID.getInstance());
+ setEnv(haWorkFlowContextID.getEnv());
+ setFlow(haWorkFlowContextID.getFlow());
+ setProject(haWorkFlowContextID.getProject());
+ setVersion(haWorkFlowContextID.getVersion());
+ setWorkSpace(haWorkFlowContextID.getWorkSpace());
+ setUser(haWorkFlowContextID.getUser());
+ }
+ }
+
+ private String nodeID;
+
+ public String getNodeID() {
+ return nodeID;
+ }
+
+ public void setNodeID(String nodeID) {
+ this.nodeID = nodeID;
+ }
+
+ public LinkisHAWorkFlowContextID getLinkisHaWorkFlowContextID() {
+ LinkisHAWorkFlowContextID linkisHAWorkFlowContextID = new LinkisHAWorkFlowContextID();
+ linkisHAWorkFlowContextID.setBackupInstance(getBackupInstance());
+ linkisHAWorkFlowContextID.setInstance(getInstance());
+ linkisHAWorkFlowContextID.setEnv(getEnv());
+ linkisHAWorkFlowContextID.setFlow(getFlow());
+ linkisHAWorkFlowContextID.setProject(getProject());
+ linkisHAWorkFlowContextID.setVersion(getVersion());
+ linkisHAWorkFlowContextID.setWorkSpace(getWorkSpace());
+ linkisHAWorkFlowContextID.setUser(getUser());
+ linkisHAWorkFlowContextID.setContextId(getContextId());
+ return linkisHAWorkFlowContextID;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextID.java
new file mode 100644
index 0000000..a0cba39
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextID.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * created by cooperyang on 2020/2/25
+ * Description:
+ */
+public class CommonContextID implements ContextID{
+
+ private String contextId;
+
+ private int contextIDType;
+
+ public CommonContextID(){
+ this.contextId = "helloworld";
+ }
+
+ @Override
+ public String getContextId() {
+ return this.contextId;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextId = contextId;
+ }
+
+ @Override
+ public int getContextIDType() {
+ return this.contextIDType;
+ }
+
+ @Override
+ public void setContextIDType(int contextIDType) {
+ this.contextIDType = contextIDType;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextKey.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextKey.java
new file mode 100644
index 0000000..844938b
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextKey.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+
+/**
+ * created by cooperyang on 2020/2/26
+ * Description:
+ */
+public class CommonContextKey implements ContextKey{
+
+ private String key;
+
+ private ContextType contextType;
+
+ private ContextScope contextScope;
+
+ private String keywords;
+
+
+ @Override
+ public String getKey() {
+ return this.key;
+ }
+
+ @Override
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+ @Override
+ public void setType(int type) {
+
+ }
+
+ @Override
+ public ContextType getContextType() {
+ return this.contextType;
+ }
+
+ @Override
+ public void setContextType(ContextType contextType) {
+ this.contextType = contextType;
+ }
+
+ @Override
+ public ContextScope getContextScope() {
+ return this.contextScope;
+ }
+
+ @Override
+ public void setContextScope(ContextScope contextScope) {
+ this.contextScope = contextScope;
+ }
+
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextKeyValue.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextKeyValue.java
new file mode 100644
index 0000000..c2a5d52
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextKeyValue.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * created by cooperyang on 2020/2/23
+ * Description:
+ */
+public class CommonContextKeyValue implements ContextKeyValue{
+
+ private ContextKey contextKey;
+
+ private ContextValue contextValue;
+
+ public CommonContextKeyValue() {}
+
+ public CommonContextKeyValue(ContextKey contextKey, ContextValue contextValue) {
+ this.contextKey = contextKey;
+ this.contextValue = contextValue;
+ }
+
+ @Override
+ public ContextKey getContextKey() {
+ return this.contextKey;
+ }
+
+ @Override
+ public void setContextKey(ContextKey contextKey) {
+ this.contextKey = contextKey;
+ }
+
+ @Override
+ public ContextValue getContextValue() {
+ return this.contextValue;
+ }
+
+ @Override
+ public void setContextValue(ContextValue contextValue) {
+ this.contextValue = contextValue;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextValue.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextValue.java
new file mode 100644
index 0000000..1408984
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonContextValue.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextValueType;
+
+/**
+ * created by cooperyang on 2020/2/23
+ * Description:
+ */
+public class CommonContextValue implements ContextValue{
+
+
+
+
+ private Object value;
+
+ private String keywords;
+
+
+
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+
+ @Override
+ public Object getValue() {
+ return this.value;
+ }
+
+ @Override
+ public void setValue(Object value) {
+ this.value = value;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonHAContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonHAContextID.java
new file mode 100644
index 0000000..ab478e9
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CommonHAContextID.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+
+public class CommonHAContextID implements HAContextID {
+
+ private String mainInstance;
+ private String backupInstance;
+ private String contextID;
+
+ public CommonHAContextID() {}
+
+ public CommonHAContextID(String mainInstance, String backupInstance, String contextID) {
+ this.mainInstance = mainInstance;
+ this.backupInstance = backupInstance;
+ this.contextID = contextID;
+ }
+
+ @Override
+ public String getInstance() {
+ return mainInstance;
+ }
+
+ @Override
+ public void setInstance(String instance) {
+ this.mainInstance = instance;
+ }
+
+ @Override
+ public String getBackupInstance() {
+ return backupInstance;
+ }
+
+ @Override
+ public void setBackupInstance(String backupInstance) {
+ this.backupInstance = backupInstance;
+ }
+
+ @Override
+ public String getContextId() {
+ return contextID;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextID = contextId;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextID.java
new file mode 100644
index 0000000..f9dedd7
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextID.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextID extends Source {
+
+ String getContextId();
+
+ void setContextId(String contextId);
+
+ default int getContextIDType(){
+ return 0;
+ }
+
+ default void setContextIDType(int contextIDType){
+
+ }
+
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextIDParser.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextIDParser.java
new file mode 100644
index 0000000..f19c067
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextIDParser.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/26
+ */
+public interface ContextIDParser {
+
+ List<String> parse(String contextId) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextKey.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextKey.java
new file mode 100644
index 0000000..b72d8a0
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextKey.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextKeyType;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextKey {
+
+ String getKey();
+
+ void setKey(String key);
+
+
+ /**
+ * 每一个ContextKey都会有一个type,比如是一个ymlContextKey的类型，这个是在ContextKeyEnum中进行枚举
+ * 这里设置这样的enum，是为了方便client和server进行交互的时候，client只需要出传一个int 就可以
+ * @return
+ */
+ int getType();
+
+ void setType(int type);
+
+
+ ContextType getContextType();
+
+ void setContextType(ContextType contextType);
+
+ ContextScope getContextScope();
+
+ void setContextScope(ContextScope contextScope);
+
+ String getKeywords();
+
+ void setKeywords(String keywords);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextKeyValue.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextKeyValue.java
new file mode 100644
index 0000000..47eb750
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextKeyValue.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextKeyValue {
+
+ ContextKey getContextKey();
+
+ void setContextKey(ContextKey contextKey);
+
+ ContextValue getContextValue();
+
+ void setContextValue(ContextValue contextValue);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextValue.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextValue.java
new file mode 100644
index 0000000..17ef71a
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ContextValue.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextValueType;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextValue {
+
+ String getKeywords();
+
+ void setKeywords(String keywords);
+
+ Object getValue();
+
+ void setValue(Object value);
+
+
+
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CreatorContextKey.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CreatorContextKey.java
new file mode 100644
index 0000000..3febdd4
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/CreatorContextKey.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface CreatorContextKey extends ContextKey {
+
+ String getCreator();
+
+ void setCreator(String creator);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/HAContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/HAContextID.java
new file mode 100644
index 0000000..0c75e5c
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/HAContextID.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface HAContextID extends ContextID {
+
+ String getInstance();
+
+ void setInstance(String instance);
+
+ // todo remain to return list
+ String getBackupInstance();
+
+ void setBackupInstance(String backupInstance);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/LinkisHAWorkFlowContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/LinkisHAWorkFlowContextID.java
new file mode 100644
index 0000000..b2f169c
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/LinkisHAWorkFlowContextID.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * created by cooperyang on 2020/2/27
+ * Description:
+ */
+public class LinkisHAWorkFlowContextID extends LinkisWorkflowContextID implements HAContextID, UserContextID {
+
+ private String instance;
+
+ private String backupInstance;
+
+ private String user;
+
+ @Override
+ public String getInstance() {
+ return this.instance;
+ }
+
+ @Override
+ public void setInstance(String instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public String getBackupInstance() {
+ return this.backupInstance;
+ }
+
+ @Override
+ public void setBackupInstance(String backupInstance) {
+ this.backupInstance = backupInstance;
+ }
+
+ @Override
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ @Override
+ public String getUser() {
+ return this.user;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/LinkisWorkflowContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/LinkisWorkflowContextID.java
new file mode 100644
index 0000000..77b3740
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/LinkisWorkflowContextID.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextIDType;
+
+/**
+ * created by cooperyang on 2020/2/27
+ * Description:
+ */
+public class LinkisWorkflowContextID implements WorkflowContextID{
+
+ private String workspace;
+
+ private String project;
+
+ private String flow;
+
+ private String contextId;
+
+ private String version;
+
+ private String env;
+
+ @Override
+ public String getWorkSpace() {
+ return this.workspace;
+ }
+
+ @Override
+ public void setWorkSpace(String workSpace) {
+ this.workspace = workSpace;
+ }
+
+ @Override
+ public String getProject() {
+ return this.project;
+ }
+
+ @Override
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ @Override
+ public String getFlow() {
+ return this.flow;
+ }
+
+ @Override
+ public void setFlow(String flow) {
+ this.flow = flow;
+ }
+
+ @Override
+ public String getVersion() {
+ return this.version;
+ }
+
+ @Override
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ @Override
+ public String getEnv() {
+ return this.env;
+ }
+
+ @Override
+ public void setEnv(String env) {
+ this.env = env;
+ }
+
+ @Override
+ public String getContextId() {
+ return this.contextId;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextId = contextId;
+ }
+
+ @Override
+ public int getContextIDType() {
+ return ContextIDType.LINKIS_WORKFLOW_CONTEXT_ID_TYPE.getIndex();
+ }
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/OrderContextKey.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/OrderContextKey.java
new file mode 100644
index 0000000..780312a
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/OrderContextKey.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface OrderContextKey extends ContextKey {
+
+ Integer getOrder();
+
+ void setOrder(Integer order);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/Source.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/Source.java
new file mode 100644
index 0000000..5c42552
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/Source.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface Source {
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/UserContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/UserContextID.java
new file mode 100644
index 0000000..c5d6159
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/UserContextID.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface UserContextID extends ContextID{
+
+ void setUser(String user);
+
+ String getUser();
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ValueBean.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ValueBean.java
new file mode 100644
index 0000000..b710497
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/ValueBean.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+
+/**
+ * created by cooperyang on 2020/2/23
+ * Description:
+ */
+public interface ValueBean {
+ public ContextType getContextType();
+ public void setContextType(ContextType contextType);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/VersionContextKey.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/VersionContextKey.java
new file mode 100644
index 0000000..fd5bfe5
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/VersionContextKey.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface VersionContextKey extends ContextKey {
+
+ String getVersion();
+
+ void setVersion(String version);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/WorkflowContextID.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/WorkflowContextID.java
new file mode 100644
index 0000000..5aa2b4d
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/WorkflowContextID.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface WorkflowContextID extends ContextID{
+
+ String getWorkSpace();
+
+ void setWorkSpace(String workSpace);
+
+ String getProject();
+
+ void setProject(String project);
+
+ String getFlow();
+
+ void setFlow(String flow);
+
+ String getVersion();
+
+ void setVersion(String version);
+
+ String getEnv();
+
+ void setEnv(String env);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/YMLContextKey.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/YMLContextKey.java
new file mode 100644
index 0000000..a541d27
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/entity/source/YMLContextKey.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.entity.source;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface YMLContextKey extends ContextKey {
+
+ void setKey(String[] key);
+
+ void setSuffix(String suffix);
+
+ void setPrefix(String[] prefix);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/CSErrorException.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/CSErrorException.java
new file mode 100644
index 0000000..c0ba387
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/CSErrorException.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.exception;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public class CSErrorException extends ErrorException {
+
+ public CSErrorException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+ public CSErrorException(int errCode, String desc, Throwable e){
+ super(errCode,desc);
+ this.initCause(e);
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/CSWarnException.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/CSWarnException.java
new file mode 100644
index 0000000..664d882
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/CSWarnException.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.exception;
+
+import com.webank.wedatasphere.linkis.common.exception.WarnException;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public class CSWarnException extends WarnException {
+
+ public CSWarnException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+
+ public CSWarnException(int errCode, String desc, Throwable e) {
+ super(errCode, desc);
+ this.initCause(e);
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/ErrorCode.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/ErrorCode.java
new file mode 100644
index 0000000..3bd8513
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/exception/ErrorCode.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.exception;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/21
+ */
+public class ErrorCode {
+
+ public static final int INVALID_NULL_OBJECT = 70100;
+
+ public static final int SERIALIZER_TO_JSON_ERROR = 70101;
+
+ public static final int INVALID_NULL_STRING = 70102;
+
+ public static final int INVALID_DESERIALIZE_STRING = 70103;
+
+ public static final int INVALID_DESERIALIZE_OBJECT = 70104;
+
+ public static final int DESERIALIZER_FROM_JSON_ERROR = 70105;
+
+ public static final int METHOD_NOT_OVERRIDE = 70106;
+
+ public static final int INVALID_HAID_ENCODE_PARAMS = 70107;
+
+ public static final int INVALID_HAID_STRING = 70108;
+
+ public static final int INVALID_CONTEXT_TYPE = 70109;
+
+ public static final int GET_CONTEXT_VALUE_ERROR = 70110;
+
+ public static final int SEARCH_CONTEXT_VALUE_ERROR = 70111;
+
+ public static final int INVALID_CONTEXT_VALUE_TYPE = 70109;
+
+
+ public static final int DESERIALIZE_ERROR = 70112;
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/listener/ContextIDListener.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/listener/ContextIDListener.java
new file mode 100644
index 0000000..045ee18
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/listener/ContextIDListener.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.listener;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextIDListener {
+
+ void onRemoved(ContextID contextID);
+
+ void onReset(ContextID contextID);
+
+ void onUPdated(ContextID contextID);
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/listener/ContextKeyListener.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/listener/ContextKeyListener.java
new file mode 100644
index 0000000..c553cc8
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/listener/ContextKeyListener.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.listener;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextKeyListener {
+
+ void onUpdated(ContextID contextID, ContextKeyValue contextKeyValue);
+
+ void onRemoved(ContextID contextID, ContextKeyValue contextKeyValue);
+
+ void onReset(ContextID contextID, ContextKeyValue contextKeyValue);
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextHTTPConstant.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextHTTPConstant.java
new file mode 100644
index 0000000..831c6bc
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextHTTPConstant.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.protocol;
+
+/**
+ * created by cooperyang on 2020/2/23
+ * Description:
+ */
+public interface ContextHTTPConstant {
+
+ String CONTEXT_KEY_STR = "contextKey";
+
+ String CONTEXT_VALUE_STR = "contextValue";
+
+ String CONTEXT_ID_STR = "contextID";
+
+ String CONTEXT_KEY_TYPE_STR = "contextKeyType";
+
+ String CONTEXT_VALUE_CLASS_NAME = "__CLASS_NAME__";
+
+ String VALUE_STR = "value";
+
+ String PROJECT_NAME_STR = "projectName";
+
+ String FLOW_NAME_STR = "flowName";
+
+ String VALUE_BEAN_STR = "valueBean";
+
+ String CONTEXT_HISTORY_KEY = "contextHistory";
+
+ String CONTEXT_KEY_PREFIX_STR = "keyPrefix";
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextHistoryType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextHistoryType.java
new file mode 100644
index 0000000..1cac740
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextHistoryType.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.protocol;
+
+/**
+ * Created by patinousward on 2020/2/27.
+ */
+public enum ContextHistoryType {
+ /**
+ *
+ */
+ COMMON_CONTEXT_HISTORY_TYPE(0, "com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextID");
+
+ private int index;
+ private String typeName;
+
+ private ContextHistoryType(int index, String typeName) {
+ this.index = index;
+ this.typeName = typeName;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextIDType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextIDType.java
new file mode 100644
index 0000000..a898254
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextIDType.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.protocol;
+
+/**
+ * created by cooperyang on 2020/2/27
+ * Description:
+ */
+public enum ContextIDType {
+ /**
+ * index表示contextValueType的int
+ * typeName 表示全路径类名
+ */
+ COMMON_CONTEXT_ID_TYPE(0, "com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextID"),
+ LINKIS_WORKFLOW_CONTEXT_ID_TYPE(1, "com.webank.wedatasphere.linkis.cs.common.entity.source.LinkisWorkflowContextID")
+ ;
+
+ private int index;
+ private String typeName;
+
+ private ContextIDType(int index, String typeName){
+ this.index = index;
+ this.typeName = typeName;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+
+ public static String getTypeName(int index){
+ for (ContextIDType type: ContextIDType.values()){
+ if (type.index == index){
+ return type.typeName;
+ }
+ }
+ return LINKIS_WORKFLOW_CONTEXT_ID_TYPE.typeName;
+ }
+
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextKeyType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextKeyType.java
new file mode 100644
index 0000000..97c51c5
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextKeyType.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.protocol;
+
+/**
+ * created by cooperyang on 2020/2/23
+ * Description:
+ */
+public enum ContextKeyType {
+
+ /**
+ * index表示type的int
+ * typeName 表示全路径类名
+ */
+ YML_CONTEXT_KEY(1, "com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey")
+ ;
+
+ private int index;
+ private String typeName;
+
+ private ContextKeyType(int index, String typeName){
+ this.index = index;
+ this.typeName = typeName;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextKeyValueType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextKeyValueType.java
new file mode 100644
index 0000000..569c60f
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextKeyValueType.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.protocol;
+
+/**
+ * Created by patinousward on 2020/2/27.
+ */
+public enum ContextKeyValueType {
+
+ /**
+ *
+ */
+ COMMON_CONTEXT_KV_TYPE(0, "com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextID");
+
+ private int index;
+ private String typeName;
+
+ private ContextKeyValueType(int index, String typeName) {
+ this.index = index;
+ this.typeName = typeName;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextValueType.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextValueType.java
new file mode 100644
index 0000000..ec3a61d
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/protocol/ContextValueType.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.protocol;
+
+/**
+ * created by cooperyang on 2020/2/27
+ * Description:
+ */
+public enum ContextValueType {
+ /**
+ * index表示contextValueType的int
+ * typeName 表示全路径类名
+ */
+ COMMON_CONTEXT_VALUE_TYPE(1, "com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue")
+ ;
+
+ private int index;
+ private String typeName;
+
+ private ContextValueType(int index, String typeName){
+ this.index = index;
+ this.typeName = typeName;
+ }
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/search/ContextSearchConditionMapBuilder.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/search/ContextSearchConditionMapBuilder.java
new file mode 100644
index 0000000..cbd4b26
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/search/ContextSearchConditionMapBuilder.java
@@ -0,0 +1,185 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.search;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+
+import java.util.List;
+import java.util.Map;
+
+public class ContextSearchConditionMapBuilder {
+
+ Map<Object, Object> contextTypeMap;
+ Map<Object, Object> contextValueTypeMap;
+ Map<Object, Object> contextScopeMap;
+ Map<Object, Object> regexMap;
+ Map<Object, Object> containsValueMap;
+ boolean nearest = false;
+ String nearestNode = "";
+ Integer nearestNumber = 1;
+ Boolean nearestUpstreamOnly = false;
+
+
+ public static ContextSearchConditionMapBuilder newBuilder(){
+ return new ContextSearchConditionMapBuilder();
+ }
+
+ public Map<Object, Object> build(){
+ Map<Object, Object> conditionMap = null;
+ if(contextTypeMap != null){
+ conditionMap = contextTypeMap;
+ }
+ if(contextValueTypeMap != null){
+ conditionMap = conditionMap == null? contextValueTypeMap : and(conditionMap, contextValueTypeMap);
+ }
+ if(contextScopeMap != null){
+ conditionMap = conditionMap == null? contextScopeMap : and(conditionMap, contextScopeMap);
+ }
+ if(regexMap != null){
+ conditionMap = conditionMap == null? regexMap : and(conditionMap, regexMap);
+ }
+ if(containsValueMap != null){
+ conditionMap = conditionMap == null? containsValueMap : and(conditionMap, containsValueMap);
+ }
+ if(nearest){
+ conditionMap = nearest(conditionMap, nearestNode, nearestNumber, nearestUpstreamOnly);
+ }
+ return conditionMap;
+ }
+
+ public ContextSearchConditionMapBuilder contextTypes(ContextType... contextTypes){
+ for(ContextType contextType :contextTypes){
+ if(contextTypeMap == null){
+ contextTypeMap = getByContextType(contextType);
+ } else {
+ contextTypeMap = or(contextTypeMap, getByContextType(contextType));
+ }
+ }
+ return this;
+ }
+
+ public ContextSearchConditionMapBuilder contextValueTypes(Class... contextValueTypes){
+ for(Class contextValueType :contextValueTypes){
+ if(contextValueTypeMap == null){
+ contextValueTypeMap = getByContextValueType(contextValueType);
+ } else {
+ contextValueTypeMap = or(contextValueTypeMap, getByContextValueType(contextValueType));
+ }
+ }
+ return this;
+ }
+
+ public ContextSearchConditionMapBuilder contextScopes(ContextScope... contextScopes){
+ for(ContextScope contextScope :contextScopes){
+ if(contextScopeMap == null){
+ contextScopeMap = getByContextScope(contextScope);
+ } else {
+ contextScopeMap = or(contextScopeMap, getByContextScope(contextScope));
+ }
+ }
+ return this;
+ }
+
+ public ContextSearchConditionMapBuilder regex(String regex){
+ regexMap = getByRegex(regex);
+ return this;
+ }
+
+ public ContextSearchConditionMapBuilder contains(String containsValue){
+ containsValueMap = getByContainsValue(containsValue);
+ return this;
+ }
+
+ public ContextSearchConditionMapBuilder nearest(String currentNode, Integer number, Boolean upstreamOnly){
+ this.nearest = true;
+ this.nearestNode = currentNode;
+ this.nearestNumber = number;
+ this.nearestUpstreamOnly = upstreamOnly;
+ return this;
+ }
+
+ private Map<Object, Object> getByContextType(ContextType contextType){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "ContextType");
+ conditionMap.put("contextType", contextType.toString());
+ return conditionMap;
+ }
+
+ private Map<Object, Object> getByContextValueType(Class contextValueType){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "ContextValueType");
+ conditionMap.put("contextValueType", contextValueType.getName());
+ return conditionMap;
+ }
+
+ private Map<Object, Object> getByContextScope(ContextScope contextScope){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "ContextScope");
+ conditionMap.put("contextScope", contextScope.toString());
+ return conditionMap;
+ }
+
+ private Map<Object, Object> getByRegex(String regex){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "Regex");
+ conditionMap.put("regex", regex);
+ return conditionMap;
+ }
+
+ private Map<Object, Object> getByContainsValue(String value){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "Contains");
+ conditionMap.put("value", value);
+ return conditionMap;
+ }
+
+ public static Map<Object, Object> nearest(Map<Object, Object> origin, String currentNode, Integer number, Boolean upstreamOnly){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "Nearest");
+ conditionMap.put("origin", origin);
+ conditionMap.put("currentNode", currentNode);
+ conditionMap.put("number", number);
+ conditionMap.put("upstreamOnly", upstreamOnly);
+ return conditionMap;
+ }
+
+ public static Map<Object, Object> not(Map<Object, Object> origin){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "Not");
+ conditionMap.put("origin", origin);
+ return conditionMap;
+ }
+
+ public static Map<Object, Object> and(Map<Object, Object> left, Map<Object, Object> right){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "And");
+ conditionMap.put("left", left);
+ conditionMap.put("right", right);
+ return conditionMap;
+ }
+
+ public static Map<Object, Object> or(Map<Object, Object> left, Map<Object, Object> right){
+ Map<Object, Object> conditionMap = Maps.newHashMap();
+ conditionMap.put("type", "Or");
+ conditionMap.put("left", left);
+ conditionMap.put("right", right);
+ return conditionMap;
+ }
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/AbstractSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/AbstractSerializer.java
new file mode 100644
index 0000000..d2905b9
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/AbstractSerializer.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/19
+ */
+public abstract class AbstractSerializer<T> implements ContextSerializer<T> {
+
+
+
+
+
+ public String getJsonValue(T t) throws CSErrorException {
+ if (null != t) {
+ return CSCommonUtils.gson.toJson(t);
+ }
+ return null;
+ }
+
+ public abstract T fromJson(String json) throws CSErrorException;
+
+
+ @Override
+ public boolean accepts(String json) {
+ if (StringUtils.isNotBlank(json)) {
+ Map<String, String> value = CSCommonUtils.gson.fromJson(json, new HashMap<String, String>().getClass());
+ if (getType().equals(value.get("type"))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+
+ @Override
+ public String serialize(T t) throws CSErrorException {
+
+ if (accepts(t)) {
+ Map<String, String> map = new HashMap<>();
+ map.put("type", getType());
+ map.put("value", getJsonValue(t));
+ return CSCommonUtils.gson.toJson(map);
+ }
+ return null;
+ }
+
+ @Override
+ public T deserialize(String json) throws CSErrorException {
+ if (accepts(json)) {
+ Map<String, String> jsonObj = CSCommonUtils.gson.fromJson(json, new HashMap<String, String>().getClass());
+ String value = jsonObj.get("value");
+ return fromJson(value);
+ }
+ return null;
+ }
+
+ @Override
+ public boolean isType(String type) {
+ return getType().equals(type);
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/ContextSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/ContextSerializer.java
new file mode 100644
index 0000000..fca7adb
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/ContextSerializer.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/19
+ */
+public interface ContextSerializer<T> {
+
+ String getType();
+
+ boolean accepts(String json);
+
+ boolean accepts(Object obj);
+
+ boolean isType(String type);
+
+ String serialize(T t) throws CSErrorException;
+
+ T deserialize(String json) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/AbstractSerializationHelper.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/AbstractSerializationHelper.java
new file mode 100644
index 0000000..500eb19
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/AbstractSerializationHelper.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.helper;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.ContextSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 19:15
+ */
+public abstract class AbstractSerializationHelper implements SerializationHelper {
+
+ protected abstract Map<String, ContextSerializer> getContextSerializerMap();
+
+ @Override
+ public boolean accepts(String json) {
+ return null != getContextSerializer(json);
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ return null != getContextSerializer(obj);
+ }
+
+ @Override
+ public String serialize(Object obj) throws CSErrorException {
+ ContextSerializer contextSerializer = getContextSerializer(obj);
+ if (null != contextSerializer) {
+ return contextSerializer.serialize(obj);
+ }
+
+ if (null != obj){
+ throw new CSErrorException(97001, "Failed to find Serializer of " + obj.getClass().getName());
+ }
+ throw new CSErrorException(97001, "The obj not null");
+ }
+
+ @Override
+ public Object deserialize(String json) throws CSErrorException {
+
+ ContextSerializer contextSerializer = getContextSerializer(json);
+
+ if (contextSerializer != null) {
+ return contextSerializer.deserialize(json);
+ }
+ if (StringUtils.isNotBlank(json)){
+ throw new CSErrorException(97001, "Failed to find deserialize of " + json);
+ }
+ throw new CSErrorException(97001, "The json not null");
+ }
+
+ @Override
+ public <T> T deserialize(String s, Class<T> interfaceClass) throws CSErrorException {
+ return null;
+ }
+
+ private ContextSerializer getContextSerializer(String json) {
+
+ if (StringUtils.isNotBlank(json)) {
+ Map<String, String> value = CSCommonUtils.gson.fromJson(json, new HashMap<String, String>().getClass());
+ String type = value.get("type");
+ return getContextSerializerMap().get(type);
+
+ }
+ return null;
+ }
+
+ private ContextSerializer getContextSerializer(Object obj) {
+
+ if (null != obj ) {
+ Stream<ContextSerializer> contextSerializerStream = getContextSerializerMap().values().stream().filter(contextSerializer -> contextSerializer.accepts(obj));
+ if (null != contextSerializerStream ) {
+ Optional<ContextSerializer> first = contextSerializerStream.findFirst();
+ if (first.isPresent()){
+ return first.get();
+ }
+ }
+ }
+ return null;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/ContextSerializationHelper.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/ContextSerializationHelper.java
new file mode 100644
index 0000000..16f613b
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/ContextSerializationHelper.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.helper;
+
+import com.webank.wedatasphere.linkis.cs.common.serialize.ContextSerializer;
+import org.reflections.Reflections;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 19:14
+ */
+public class ContextSerializationHelper extends AbstractSerializationHelper {
+
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextSerializationHelper.class);
+
+ private Map<String, ContextSerializer> contextSerializerMap = new HashMap<>(16);
+
+
+ private void init() {
+ Reflections reflections = new Reflections("com.webank.wedatasphere.linkis", ContextSerializationHelper.class.getClassLoader());
+ Set<Class<? extends ContextSerializer>> allSubClass = reflections.getSubTypesOf(ContextSerializer.class);
+
+ if ( null != allSubClass){
+ Iterator<Class<? extends ContextSerializer>> iterator = allSubClass.iterator();
+ while (iterator.hasNext()){
+ Class<? extends ContextSerializer> next = iterator.next();
+ ContextSerializer contextSerializer = null;
+ try {
+ contextSerializer = next.newInstance();
+ } catch (InstantiationException e) {
+ logger.info("Failed to Instantiation " + next.getName());
+ continue;
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Failed to construct contextSerializer", e);
+ }
+
+ if (contextSerializerMap.containsKey(contextSerializer.getType())){
+ throw new RuntimeException("contextSerializer Type cannot be duplicated ");
+ }
+ contextSerializerMap.put(contextSerializer.getType(), contextSerializer);
+ }
+ }
+ }
+
+
+
+ private static ContextSerializationHelper contextSerializationHelper = null;
+
+ public static ContextSerializationHelper getInstance() {
+ if (contextSerializationHelper == null) {
+ synchronized (ContextSerializationHelper.class) {
+ if (contextSerializationHelper == null) {
+ contextSerializationHelper = new ContextSerializationHelper();
+ contextSerializationHelper.init();
+ }
+ }
+ }
+ return contextSerializationHelper;
+ }
+
+ @Override
+ protected Map<String, ContextSerializer> getContextSerializerMap() {
+ return this.contextSerializerMap;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/SerializationHelper.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/SerializationHelper.java
new file mode 100644
index 0000000..a93faa2
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/helper/SerializationHelper.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.helper;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 19:03
+ */
+public interface SerializationHelper {
+
+ boolean accepts(String json);
+
+ boolean accepts(Object obj);
+
+ String serialize(Object obj) throws CSErrorException;
+
+ Object deserialize(String json) throws CSErrorException;
+
+ <T> T deserialize(String s, Class<T> interfaceClass) throws CSErrorException;
+
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/CombinedNodeIDContextIDSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/CombinedNodeIDContextIDSerializer.java
new file mode 100644
index 0000000..374ed69
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/CombinedNodeIDContextIDSerializer.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.context;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CombinedNodeIDContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/3/10 23:30
+ */
+public class CombinedNodeIDContextIDSerializer extends AbstractSerializer<CombinedNodeIDContextID> {
+
+
+ @Override
+ public String getType() {
+ return "CombinedNodeIDContextID";
+ }
+
+
+
+
+ @Override
+ public CombinedNodeIDContextID fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, CombinedNodeIDContextID.class);
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+
+ if (null != obj && obj instanceof CombinedNodeIDContextID){
+ return true;
+ }
+ return false;
+ }
+
+
+}
+
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/CommonContextKeySerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/CommonContextKeySerializer.java
new file mode 100644
index 0000000..68b10a4
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/CommonContextKeySerializer.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.context;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 23:26
+ */
+public class CommonContextKeySerializer extends AbstractSerializer<CommonContextKey> {
+
+
+ @Override
+ public CommonContextKey fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, CommonContextKey.class);
+ }
+
+ @Override
+ public String getType() {
+ return "commonContextKey";
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj instanceof CommonContextKey){
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/ContextKeyValueSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/ContextKeyValueSerializer.java
new file mode 100644
index 0000000..7b72784
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/ContextKeyValueSerializer.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.context;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 23:16
+ */
+public class ContextKeyValueSerializer extends AbstractSerializer<CommonContextKeyValue> {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextKeyValueSerializer.class);
+
+ @Override
+ public String getJsonValue(CommonContextKeyValue contextKeyValue) throws CSErrorException {
+ try {
+ Map<String, String> map = new HashMap<>();
+ ContextKey contextKey = contextKeyValue.getContextKey();
+ ContextValue contextValue = contextKeyValue.getContextValue();
+ map.put("key", ContextSerializationHelper.getInstance().serialize(contextKey));
+ map.put("value", ContextSerializationHelper.getInstance().serialize(contextValue));
+ return CSCommonUtils.gson.toJson(map);
+ } catch (Exception e) {
+ logger.error("Failed to serialize contextKeyValue: ", e);
+ throw new CSErrorException(97000, "Failed to serialize contextKeyValue");
+ }
+ }
+
+ @Override
+ public CommonContextKeyValue fromJson(String json) throws CSErrorException {
+ try {
+ Map<String, String> jsonObj = CSCommonUtils.gson.fromJson(json, new HashMap<String, String>().getClass());
+ String key = jsonObj.get("key");
+ String value = jsonObj.get("value");
+ Object contextKey = ContextSerializationHelper.getInstance().deserialize(key);
+ Object contextValue = ContextSerializationHelper.getInstance().deserialize(value);
+ CommonContextKeyValue contextKeyValue = new CommonContextKeyValue();
+ contextKeyValue.setContextKey((ContextKey) contextKey);
+ contextKeyValue.setContextValue((ContextValue) contextValue);
+ return contextKeyValue;
+ } catch (Exception e) {
+ logger.error("Failed to deserialize contextKeyValue: ", e);
+ throw new CSErrorException(97000, "Failed to serialize contextKeyValue");
+ }
+ }
+
+ @Override
+ public String getType() {
+ return "commonContextKeyValue";
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj instanceof CommonContextKeyValue){
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/ContextValueSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/ContextValueSerializer.java
new file mode 100644
index 0000000..9e6198d
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/ContextValueSerializer.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.context;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 21:48
+ */
+public class ContextValueSerializer extends AbstractSerializer<CommonContextValue> {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextValueSerializer.class);
+
+
+ @Override
+ public String getType() {
+ return "contextValue";
+ }
+
+ @Override
+ public String getJsonValue(CommonContextValue commonContextValue) throws CSErrorException {
+ Object value = commonContextValue.getValue();
+ try {
+ Map<String, String> map = new HashMap<>();
+ String keywords = commonContextValue.getKeywords();
+ String json = ContextSerializationHelper.getInstance().serialize(value);
+ map.put("keywords", keywords);
+ map.put("value", json);
+ return CSCommonUtils.gson.toJson(map);
+ } catch (Exception e) {
+ logger.error("Failed to serialize contextValue: ", e);
+ throw new CSErrorException(97000, "Failed to serialize contextValue");
+ }
+ }
+
+ @Override
+ public CommonContextValue fromJson(String json) throws CSErrorException {
+ try {
+ Map<String, String> jsonObj = CSCommonUtils.gson.fromJson(json, new HashMap<String, String>().getClass());
+ String value = jsonObj.get("value");
+ String keywords = jsonObj.get("keywords");
+ Object valueObj = ContextSerializationHelper.getInstance().deserialize(value);
+ CommonContextValue commonContextValue = new CommonContextValue();
+ commonContextValue.setKeywords(keywords);
+ commonContextValue.setValue(valueObj);
+ return commonContextValue;
+ } catch (Exception e) {
+ logger.error("Failed to deserialize contextValue: ", e);
+ throw new CSErrorException(97000, "Failed to serialize contextValue");
+ }
+
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj instanceof CommonContextValue){
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/LinkisHAWorkFlowContextIDSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/LinkisHAWorkFlowContextIDSerializer.java
new file mode 100644
index 0000000..8fe72fd
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/context/LinkisHAWorkFlowContextIDSerializer.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.context;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.LinkisHAWorkFlowContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 19:39
+ */
+public class LinkisHAWorkFlowContextIDSerializer extends AbstractSerializer<LinkisHAWorkFlowContextID> {
+
+
+ @Override
+ public String getType() {
+ return "HAWorkFlowContextID";
+ }
+
+
+
+
+ @Override
+ public LinkisHAWorkFlowContextID fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, LinkisHAWorkFlowContextID.class);
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj.getClass().getName().equals(LinkisHAWorkFlowContextID.class.getName())){
+ return true;
+ }
+ return false;
+ }
+
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/LinkisBMLResourceSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/LinkisBMLResourceSerializer.java
new file mode 100644
index 0000000..33e1ba3
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/LinkisBMLResourceSerializer.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.value;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 22:21
+ */
+public class LinkisBMLResourceSerializer extends AbstractSerializer<LinkisBMLResource> {
+ @Override
+ public String getType() {
+ return "linkisBMLResource";
+ }
+
+
+ @Override
+ public LinkisBMLResource fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, LinkisBMLResource.class);
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj instanceof LinkisBMLResource){
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/data/CSResultDataSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/data/CSResultDataSerializer.java
new file mode 100644
index 0000000..0066b0a
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/data/CSResultDataSerializer.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.value.data;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.data.CSResultData;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 19:35
+ */
+public class CSResultDataSerializer extends AbstractSerializer<CSResultData> {
+ @Override
+ public CSResultData fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, CSResultData.class);
+ }
+
+ @Override
+ public String getType() {
+ return "CSResultData";
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj.getClass().getName().equals(CSResultData.class.getName())){
+ return true;
+ }
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/data/LinkisJobDataSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/data/LinkisJobDataSerializer.java
new file mode 100644
index 0000000..bb01f93
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/data/LinkisJobDataSerializer.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.value.data;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.data.LinkisJobData;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 22:10
+ */
+public class LinkisJobDataSerializer extends AbstractSerializer<LinkisJobData> {
+
+ @Override
+ public LinkisJobData fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, LinkisJobData.class);
+ }
+
+ @Override
+ public String getType() {
+ return "LinkisJobData";
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj.getClass().getName().equals(LinkisJobData.class.getName())) {
+ return true;
+ }
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/metadata/CSTableSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/metadata/CSTableSerializer.java
new file mode 100644
index 0000000..6d8d468
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/metadata/CSTableSerializer.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.value.metadata;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/3/17 19:33
+ */
+public class CSTableSerializer extends AbstractSerializer<CSTable> {
+
+ @Override
+ public CSTable fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, CSTable.class);
+ }
+
+ @Override
+ public String getType() {
+ return "CSTable";
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj.getClass().getName().equals(CSTable.class.getName())) {
+ return true;
+ }
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/object/CSFlowInfosSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/object/CSFlowInfosSerializer.java
new file mode 100644
index 0000000..9c55ef9
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/object/CSFlowInfosSerializer.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.value.object;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/3/11 17:40
+ */
+public class CSFlowInfosSerializer extends AbstractSerializer<CSFlowInfos> {
+ @Override
+ public CSFlowInfos fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, CSFlowInfos.class);
+ }
+
+ @Override
+ public String getType() {
+ return "CSFlowInfos";
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj.getClass().getName().equals(CSFlowInfos.class.getName())){
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/object/LinkisVariableSerializer.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/object/LinkisVariableSerializer.java
new file mode 100644
index 0000000..cfa03e9
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/serialize/impl/value/object/LinkisVariableSerializer.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.serialize.impl.value.object;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.object.LinkisVariable;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.AbstractSerializer;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+
+/**
+ * @author peacewong
+ * @date 2020/3/17 19:35
+ */
+public class LinkisVariableSerializer extends AbstractSerializer<LinkisVariable> {
+ @Override
+ public LinkisVariable fromJson(String json) throws CSErrorException {
+ return CSCommonUtils.gson.fromJson(json, LinkisVariable.class);
+ }
+
+ @Override
+ public String getType() {
+ return "LinkisVariable";
+ }
+
+ @Override
+ public boolean accepts(Object obj) {
+ if (null != obj && obj.getClass().getName().equals(LinkisVariable.class.getName())){
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/utils/CSCommonUtils.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/utils/CSCommonUtils.java
new file mode 100644
index 0000000..c6fc4ad
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/utils/CSCommonUtils.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.utils;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * @author peacewong
+ * @date 2020/2/27 20:56
+ */
+public class CSCommonUtils {
+
+ public final static String CONTEXT_ID_STR = "contextID";
+
+ public final static String NODE_NAME_STR = "nodeName";
+
+ public final static String NODE_ID = "nodeID";
+
+ public final static String ID_NODE_NAME = "id_nodeName";
+
+ public final static String FLOW_INFOS = "flow.infos";
+
+
+ public final static String CONTEXT_ENV_DEV = CommonVars.apply("wds.linkis.dev.contextID.env", "BDP_DEV").getValue();
+
+ public final static String CONTEXT_ENV_PROD = CommonVars.apply("wds.linkis.production.contextID.env", "BDP_PRODUCTION").getValue();
+
+ public final static String CS_TMP_TABLE_PREFIX = "cs_tmp_";
+
+ public static Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls().create();
+
+
+ public static final String NODE_PREFIX = "node.";
+
+ public static final String FLOW_PREFIX = "flow.";
+
+ public static final String PROJECT_PREFIX = "project.";
+
+ public static final String WORKSPACE_PREFIX = "workspace.";
+
+ public static final String RESOURCE_PREFIX = "resource.";
+
+ public static final String TABLE_PREFIX = "table.";
+
+ public static final String DB_PREFIX = "db.";
+
+ public static final String VARIABLE_PREFIX = "variable.";
+
+ public static final String JOB_ID = ".jobID";
+
+ public static final String FLOW_RESOURCE_PREFIX = FLOW_PREFIX + RESOURCE_PREFIX;
+
+ public static final String PROJECT_RESOURCE_PREFIX = PROJECT_PREFIX + RESOURCE_PREFIX;
+
+ public static final String WORKSPACE_RESOURCE_PREFIX = WORKSPACE_PREFIX + RESOURCE_PREFIX;
+
+ public static final String FLOW_VARIABLE_PREFIX = FLOW_PREFIX + VARIABLE_PREFIX;
+
+ public static final String WORKSPACE_VARIABLE_PREFIX = WORKSPACE_PREFIX + VARIABLE_PREFIX;
+
+ public static final String PROJECT_VARIABLE_PREFIX = PROJECT_PREFIX + VARIABLE_PREFIX;
+
+ public static String getVariableKey(String nodeName, String varName) {
+
+ return CSCommonUtils.NODE_PREFIX + nodeName + "." + CSCommonUtils.VARIABLE_PREFIX + varName;
+
+ }
+
+
+ public static String getTableKey(String nodeName, String tableName) {
+
+ return CSCommonUtils.NODE_PREFIX + nodeName + "." + CSCommonUtils.TABLE_PREFIX + tableName;
+
+ }
+
+}
diff --git a/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/utils/CSHighAvailableUtils.java b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/utils/CSHighAvailableUtils.java
new file mode 100644
index 0000000..ba83e40
--- /dev/null
+++ b/contextservice/cs-common/src/main/java/com/webank/wedatasphere/linkis/cs/common/utils/CSHighAvailableUtils.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.common.utils;
+
+import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonHAContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Author alexyang
+ * @Date 2020/3/3
+ */
+public class CSHighAvailableUtils {
+
+ private static final Logger logger = LoggerFactory.getLogger(CSHighAvailableUtils.class);
+ private final static String HAID_PART_DELEMETER = "--";
+ private final static String HAID_INS_LEN_DELEMETER = "-";
+ private final static int HAID_PARTS_NUM = 2;
+ private final static Gson gson = new Gson();
+
+ public static boolean checkHAIDBasicFormat(String haid) {
+ if (StringUtils.isBlank(haid)) {
+ return false;
+ }
+ String [] arr = haid.split(HAID_PART_DELEMETER);
+ if (null != arr && arr.length == HAID_PARTS_NUM) {
+ int insLen = 0;
+ String [] lenArr = arr[0].split(HAID_INS_LEN_DELEMETER);
+ if (null == lenArr || lenArr.length < 2) {
+ return false;
+ }
+ try {
+ for (String len : lenArr) {
+ insLen += Integer.parseInt(len);
+ }
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ if (insLen < arr[1].length()) {
+ String id = arr[1].substring(insLen);
+ if (StringUtils.isNumeric(id)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public static String encodeHAIDKey(String contextID, String instance, List<String> backupInstanceList) throws CSErrorException {
+ if (!StringUtils.isNumeric(contextID) || StringUtils.isBlank(instance) || null == backupInstanceList || backupInstanceList.isEmpty()) {
+ logger.error("Cannot encodeHAIDKey, contextID : " + contextID
+ + ", instance : " + instance + ", backupInstanceList : " + gson.toJson(backupInstanceList));
+ throw new CSErrorException(ErrorCode.INVALID_HAID_ENCODE_PARAMS, "Cannot encodeHAIDKey, contextID : " + contextID
+ + ", instance : " + instance + ", backupInstanceList : " + gson.toJson(backupInstanceList));
+ }
+ StringBuilder idBuilder = new StringBuilder("");
+ StringBuilder instBuilder = new StringBuilder("");
+ idBuilder.append(instance.length());
+ instBuilder.append(instance);
+ for (String ins : backupInstanceList) {
+ idBuilder.append(HAID_INS_LEN_DELEMETER).append(ins.length());
+ instBuilder.append(ins);
+ }
+ idBuilder.append(HAID_PART_DELEMETER)
+ .append(instBuilder)
+ .append(contextID);
+ return idBuilder.toString();
+ }
+
+ public static HAContextID decodeHAID(String haid) throws CSErrorException {
+ if (StringUtils.isBlank(haid)) {
+ throw new CSErrorException(ErrorCode.INVALID_NULL_STRING, "HAIDKey cannot be empty.");
+ }
+ if (!checkHAIDBasicFormat(haid)) {
+ logger.error("Invalid haid : " + haid);
+ throw new CSErrorException(ErrorCode.INVALID_HAID_STRING, "Invalid haid : " + haid);
+ }
+ String [] partArr = haid.split(HAID_PART_DELEMETER);
+ String [] insArr = partArr[0].split(HAID_INS_LEN_DELEMETER);
+ String contextID = null;
+ List<String> instanceList = new ArrayList<>();
+ String insStr = partArr[1];
+ try {
+ int index = 0, tmp = 0;
+ for (String len : insArr) {
+ tmp = Integer.parseInt(len);
+ instanceList.add(insStr.substring(index, index + tmp));
+ index += tmp;
+ }
+ contextID = insStr.substring(index);
+ } catch (NumberFormatException e) {
+ logger.error("Invalid haid : " + haid + ", " + e.getMessage());
+ throw new CSErrorException(ErrorCode.INVALID_HAID_STRING, "Invalid haid : " + haid + ", " + e.getMessage());
+ }
+ String instance = instanceList.remove(0);
+ return new CommonHAContextID(instance, instanceList.get(0), contextID);
+ }
+
+ public static void main(String [] args) throws Exception {
+ String haid1 = "24--24--YmRwaGRwMTFpZGUwMTo5MTE2YmRwaGRwMTFpZGUwMTo5MTE084835";
+ System.out.println(checkHAIDBasicFormat(haid1));
+ String id = "8798";
+ String instance = "jslfjslfjlsdjfljsdf==+";
+ String backupInstance = "sjljsljflsdjflsjd";
+ List<String> list = new ArrayList<>();
+ list.add(backupInstance);
+ list.add(instance);
+ String haid2 = encodeHAIDKey(id, instance, list);
+ System.out.println(haid2);
+ System.out.println(checkHAIDBasicFormat(haid2));
+ if (checkHAIDBasicFormat(haid2)) {
+ System.out.println(gson.toJson(decodeHAID(haid2)));
+ }
+ String haid3 = "24-24--YmRwaGRwMTFpZGUwMTo5MTE0YmRwaGRwMTFpZGUwMTo5MTE084855";
+ if (checkHAIDBasicFormat(haid3)) {
+ System.out.println(gson.toJson(decodeHAID(haid3)));
+ } else {
+ System.out.println("Invalid haid3 : " + haid3);
+ }
+ }
+}
diff --git a/contextservice/cs-highavailable/pom.xml b/contextservice/cs-highavailable/pom.xml
new file mode 100644
index 0000000..fe5cdcf
--- /dev/null
+++ b/contextservice/cs-highavailable/pom.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-highavailable</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cloudRPC</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-persistence</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.1.1</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/AbstractContextHAManager.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/AbstractContextHAManager.java
new file mode 100644
index 0000000..a409a76
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/AbstractContextHAManager.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.BackupInstanceGenerator;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.ContextHAChecker;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.ContextHAIDGenerator;
+
+/**
+ *
+ * @Author alexyang
+ * @Date 2020/2/16
+ */
+public abstract class AbstractContextHAManager implements ContextHAManager {
+
+ public abstract ContextHAIDGenerator getContextHAIDGenerator();
+
+ public abstract ContextHAChecker getContextHAChecker();
+
+ public abstract BackupInstanceGenerator getBackupInstanceGenerator();
+
+ public abstract HAContextID convertProxyHAID(HAContextID contextID) throws CSErrorException;
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ContextHAManager.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ContextHAManager.java
new file mode 100644
index 0000000..761815f
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ContextHAManager.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+/**
+ * 动态代理实现持久层HAIDKey和contextID的动态转换
+ *
+ * @Author alexyang
+ * @Date 2020/2/16
+ */
+public interface ContextHAManager {
+
+ <T> T getContextHAProxy(T persistence) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/DefaultContextHAManager.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/DefaultContextHAManager.java
new file mode 100644
index 0000000..22f7c5a
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/DefaultContextHAManager.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable;
+
+import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.highavailable.exception.ErrorCode;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.BackupInstanceGenerator;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.ContextHAChecker;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.ContextHAIDGenerator;
+import com.webank.wedatasphere.linkis.cs.highavailable.proxy.MethodInterceptorImpl;
+import net.sf.cglib.proxy.Callback;
+import net.sf.cglib.proxy.Enhancer;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * ContextService高可用管理器默认实现
+ * 采用CGLib动态代理，一般用于CS持久层存储转换，将HAContextID实例进行转换
+ * @Author alexyang
+ * @Date 2020/2/19
+ */
+@Component
+public class DefaultContextHAManager extends AbstractContextHAManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultContextHAManager.class);
+ private static final Gson gson = new Gson();
+
+ @Autowired
+ private ContextHAIDGenerator contextHAIDGenerator;
+ @Autowired
+ private ContextHAChecker contextHAChecker;
+ @Autowired
+ private BackupInstanceGenerator backupInstanceGenerator;
+
+ public DefaultContextHAManager() {}
+
+
+ @Override
+ public <T> T getContextHAProxy(T persistence) throws CSErrorException {
+ Callback callback = new MethodInterceptorImpl(this, persistence);
+ Enhancer enhancer = new Enhancer();
+ enhancer.setSuperclass(persistence.getClass());
+ Callback [] callbacks = new Callback[] {callback};
+ enhancer.setCallbacks(callbacks);
+ return (T)enhancer.create();
+ }
+
+ @Override
+ public HAContextID convertProxyHAID(HAContextID haContextID) throws CSErrorException {
+
+ if (null == haContextID) {
+ logger.error("HaContextID cannot be null.");
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "HaContextID cannot be null.");
+ }
+ if (StringUtils.isBlank(haContextID.getContextId())) {
+ // generate new haid
+ HAContextID tmpHAID = contextHAIDGenerator.generateHAContextID(null);
+ haContextID.setContextId(tmpHAID.getContextId());
+ haContextID.setInstance(tmpHAID.getInstance());
+ haContextID.setBackupInstance(tmpHAID.getBackupInstance());
+ return haContextID;
+ } else if (StringUtils.isNotBlank(haContextID.getInstance()) && StringUtils.isNotBlank(haContextID.getBackupInstance())) {
+ if (StringUtils.isNumeric(haContextID.getContextId())) {
+ // convert contextID to haID
+ String haIdKey = contextHAChecker.convertHAIDToHAKey(haContextID);
+ haContextID.setContextId(haIdKey);
+ } else if (contextHAChecker.isHAIDValid(haContextID.getContextId())) {
+ String contextID = contextHAChecker.parseHAIDFromKey(haContextID.getContextId()).getContextId();
+ haContextID.setContextId(contextID);
+ } else {
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "Invalid contextID in haContextID : " + gson.toJson(haContextID));
+ }
+ return haContextID;
+ } else {
+ // complete ha property
+ if (StringUtils.isNumeric(haContextID.getContextId())) {
+ HAContextID tmpHAID = contextHAIDGenerator.generateHAContextID(haContextID);
+ haContextID.setInstance(tmpHAID.getInstance());
+ haContextID.setBackupInstance(tmpHAID.getBackupInstance());
+ } else if (contextHAChecker.isHAIDValid(haContextID.getContextId())) {
+ HAContextID tmpHAID = contextHAChecker.parseHAIDFromKey(haContextID.getContextId());
+ haContextID.setContextId(tmpHAID.getContextId());
+ haContextID.setInstance(tmpHAID.getInstance());
+ haContextID.setBackupInstance(tmpHAID.getBackupInstance());
+ } else {
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "Invalid contextID in haContextID : " + gson.toJson(haContextID));
+ }
+ // todo debug
+ if (contextHAChecker.isHAContextIDValid(haContextID)) {
+ logger.info("HAID : " + contextHAChecker.convertHAIDToHAKey(haContextID));
+ }
+ return haContextID;
+ }
+ }
+
+ @Override
+ public ContextHAIDGenerator getContextHAIDGenerator() {
+ return contextHAIDGenerator;
+ }
+
+ @Override
+ public ContextHAChecker getContextHAChecker() {
+ return contextHAChecker;
+ }
+
+ @Override
+ public BackupInstanceGenerator getBackupInstanceGenerator() {
+ return backupInstanceGenerator;
+ }
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/conf/ContextHighAvailableConf.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/conf/ContextHighAvailableConf.java
new file mode 100644
index 0000000..a56e62b
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/conf/ContextHighAvailableConf.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.conf;
+
+public class ContextHighAvailableConf {
+
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/exception/ErrorCode.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/exception/ErrorCode.java
new file mode 100644
index 0000000..53d43bf
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/exception/ErrorCode.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.exception;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/18
+ */
+public class ErrorCode {
+
+ public static int INVALID_INSTANCE_ALIAS = 70010;
+
+ public static int INVALID_HAID = 70011;
+
+ public static int GENERATE_HAID_ERROR = 70012;
+
+ public static int INVALID_CONTEXTID = 70013;
+
+ public static int GENERATE_BACKUP_INSTANCE_ERROR = 70014;
+
+ public static int INVALID_INSTANCE = 70015;
+
+ public static int INVAID_HA_CONTEXTID = 70016;
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/BackupInstanceGenerator.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/BackupInstanceGenerator.java
new file mode 100644
index 0000000..98ffa13
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/BackupInstanceGenerator.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.ha;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+public interface BackupInstanceGenerator {
+
+ String getBackupInstance(String haID) throws CSErrorException;
+
+ String chooseBackupInstance(String mainInstance) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/ContextHAChecker.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/ContextHAChecker.java
new file mode 100644
index 0000000..547ee9d
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/ContextHAChecker.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.ha;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+public interface ContextHAChecker {
+
+ boolean isHAIDValid(String haIDKey);
+
+ boolean isHAContextIDValid(HAContextID haContextID) throws CSErrorException;
+
+ String convertHAIDToHAKey(HAContextID haContextID) throws CSErrorException;
+
+ HAContextID parseHAIDFromKey(String haIDKey) throws CSErrorException;
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/ContextHAIDGenerator.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/ContextHAIDGenerator.java
new file mode 100644
index 0000000..294efd8
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/ContextHAIDGenerator.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.ha;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+public interface ContextHAIDGenerator {
+
+ HAContextID generateHAContextID(ContextID contextID) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/BackupInstanceGeneratorImpl.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/BackupInstanceGeneratorImpl.java
new file mode 100644
index 0000000..61b08e9
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/BackupInstanceGeneratorImpl.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.ha.impl;
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.highavailable.exception.ErrorCode;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.BackupInstanceGenerator;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.ContextHAChecker;
+import com.webank.wedatasphere.linkis.rpc.instancealias.InstanceAliasManager;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+@Component
+public class BackupInstanceGeneratorImpl implements BackupInstanceGenerator {
+ private final static Logger logger = LoggerFactory.getLogger(BackupInstanceGeneratorImpl.class);
+
+ @Autowired
+ private InstanceAliasManager instanceAliasManager;
+
+ @Autowired
+ private ContextHAChecker contextHAChecker;
+
+ @Override
+ public String getBackupInstance(String haIDKey) throws CSErrorException {
+
+ String alias = null;
+ if (StringUtils.isNotBlank(haIDKey) && contextHAChecker.isHAIDValid(haIDKey)) {
+ alias = contextHAChecker.parseHAIDFromKey(haIDKey).getBackupInstance();
+ } else {
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "Invalid HAID :" + haIDKey);
+ }
+ return alias;
+ }
+
+ @Override
+ public String chooseBackupInstance(String mainInstanceAlias) throws CSErrorException {
+ ServiceInstance mainInstance = null;
+ try {
+ mainInstance = instanceAliasManager.getInstanceByAlias(mainInstanceAlias);
+ } catch (Exception e) {
+ logger.error("Get Instance error, alias : {}, message : {}", mainInstanceAlias, e.getMessage());
+ throw new CSErrorException(ErrorCode.INVALID_INSTANCE_ALIAS, e.getMessage() + ", alias : " + mainInstanceAlias);
+ }
+ List<ServiceInstance> allInstanceList = instanceAliasManager.getAllInstanceList();
+ List<ServiceInstance> remainInstanceList = new ArrayList<>();
+ for (ServiceInstance instance : allInstanceList) {
+ if (instance.equals(mainInstance)) {
+ continue;
+ }
+ remainInstanceList.add(instance);
+ }
+ if (remainInstanceList.size() > 0) {
+ int index = getBackupInstanceIndex(remainInstanceList);
+ return instanceAliasManager.getAliasByServiceInstance(remainInstanceList.get(index));
+ } else {
+ // only one service instance
+ logger.error("Only one instance, no remains.");
+ return instanceAliasManager.getAliasByServiceInstance(mainInstance);
+ }
+ }
+
+ private int getBackupInstanceIndex(List<ServiceInstance> instanceList) {
+
+ // todo refactor according to load-balance
+ return new Random().nextInt(instanceList.size());
+ }
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/ContextHACheckerImpl.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/ContextHACheckerImpl.java
new file mode 100644
index 0000000..480a2b1
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/ContextHACheckerImpl.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.ha.impl;
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication;
+import com.webank.wedatasphere.linkis.common.ServiceInstance;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonHAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSHighAvailableUtils;
+import com.webank.wedatasphere.linkis.cs.highavailable.exception.ErrorCode;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.ContextHAChecker;
+import com.webank.wedatasphere.linkis.rpc.instancealias.impl.InstanceAliasManagerImpl;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/19
+ */
+@Component
+public class ContextHACheckerImpl implements ContextHAChecker {
+
+ private final static Logger logger = LoggerFactory.getLogger(ContextHACheckerImpl.class);
+
+ @Autowired
+ private InstanceAliasManagerImpl instanceAliasManager;
+ /**
+ * ${第一个instance长度}${第二个instance长度}{instance别名1}{instance别名2}{实际ID}
+ *
+ * @param haIDKey
+ * @return
+ */
+ @Override
+ public boolean isHAIDValid(String haIDKey) {
+ if (CSHighAvailableUtils.checkHAIDBasicFormat(haIDKey)) {
+ try {
+ return checkHAIDInstance(CSHighAvailableUtils.decodeHAID(haIDKey));
+ } catch (CSErrorException e) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * 主备实例同时有效，且id为有效的HAID或数字时才有效
+ * @param haContextID
+ * @return
+ * @throws CSErrorException
+ */
+ @Override
+ public boolean isHAContextIDValid(HAContextID haContextID) throws CSErrorException {
+ boolean valid = false;
+ if (null != haContextID && StringUtils.isNotBlank(haContextID.getInstance())
+ && StringUtils.isNotBlank(haContextID.getBackupInstance())) {
+ if (StringUtils.isNotBlank(haContextID.getContextId())) {
+ if (StringUtils.isNumeric(haContextID.getContextId())) {
+ valid = checkHAIDInstance(haContextID);
+ } else {
+ valid = isHAIDValid(haContextID.getContextId());
+ }
+ } else {
+ valid = false;
+ }
+ } else {
+ valid = false;
+ }
+ return valid;
+ }
+
+ @Override
+ public String convertHAIDToHAKey(HAContextID haContextID) throws CSErrorException {
+ if (null == haContextID || StringUtils.isBlank(haContextID.getInstance())
+ || StringUtils.isBlank(haContextID.getBackupInstance())
+ || StringUtils.isBlank(haContextID.getContextId())) {
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "Incomplete HAID Object cannot be encoded. mainInstance : "
+ + haContextID.getInstance() + ", backupInstance : " + haContextID.getBackupInstance() + ", contextID : " + haContextID.getContextId());
+ }
+ if (StringUtils.isNumeric(haContextID.getContextId())) {
+ return encode(haContextID);
+ } else if (isHAIDValid(haContextID.getContextId())) {
+ return haContextID.getContextId();
+ } else {
+ logger.error("ConvertHAIDToHAKey error, invald HAID : " + haContextID.getContextId());
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "ConvertHAIDToHAKey error, invald HAID : " + haContextID.getContextId());
+ }
+ }
+
+ /**
+ * Encode HAContextID to HAKey String.
+ * ${第一个instance长度}${第二个instance长度}{instance别名0}{instance别名2}{实际ID}
+ * @return
+ */
+ private String encode(HAContextID haContextID) throws CSErrorException {
+ List<String> backupInstanceList = new ArrayList<>();
+ if (StringUtils.isNotBlank(haContextID.getBackupInstance())) {
+ backupInstanceList.add(haContextID.getBackupInstance());
+ } else {
+ backupInstanceList.add(haContextID.getInstance());
+ }
+ return CSHighAvailableUtils.encodeHAIDKey(haContextID.getContextId(), haContextID.getInstance(), backupInstanceList);
+ }
+
+ private boolean checkHAIDInstance(HAContextID haContextID) {
+ ServiceInstance serverMainInstance = DataWorkCloudApplication.getServiceInstance();
+ String mainInstanceAlias = haContextID.getInstance();
+ String backupInstanceAlias = haContextID.getBackupInstance();
+ if (!instanceAliasManager.isInstanceAliasValid(mainInstanceAlias) && !instanceAliasManager.isInstanceAliasValid(backupInstanceAlias)) {
+ return false;
+ }
+ ServiceInstance mainInstance = instanceAliasManager.getInstanceByAlias(mainInstanceAlias);
+ ServiceInstance backupInstance = instanceAliasManager.getInstanceByAlias(backupInstanceAlias);
+ if (serverMainInstance.equals(mainInstance) || serverMainInstance.equals(backupInstance)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public HAContextID parseHAIDFromKey(String haIDKey) throws CSErrorException {
+ HAContextID haContextID = null;
+ if (StringUtils.isBlank(haIDKey) || !CSHighAvailableUtils.checkHAIDBasicFormat(haIDKey)) {
+ logger.error("Invalid haIDKey : " + haIDKey);
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "Invalid haIDKey : " + haIDKey);
+ }
+ return CSHighAvailableUtils.decodeHAID(haIDKey);
+ }
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/ContextHAIDGeneratorImpl.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/ContextHAIDGeneratorImpl.java
new file mode 100644
index 0000000..a843044
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/ha/impl/ContextHAIDGeneratorImpl.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.ha.impl;
+
+import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication;
+import com.webank.wedatasphere.linkis.common.ServiceInstance;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonHAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.highavailable.exception.ErrorCode;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.BackupInstanceGenerator;
+import com.webank.wedatasphere.linkis.cs.highavailable.ha.ContextHAIDGenerator;
+import com.webank.wedatasphere.linkis.rpc.instancealias.InstanceAliasConverter;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ContextHAIDGeneratorImpl implements ContextHAIDGenerator {
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextHAIDGeneratorImpl.class);
+
+ @Autowired
+ private BackupInstanceGenerator backupInstanceGenerator;
+
+ @Autowired
+ private InstanceAliasConverter instanceAliasConverter;
+
+ @Override
+ public HAContextID generateHAContextID(ContextID contextID) throws CSErrorException {
+ String contextIDKey = null;
+ if (null != contextID && StringUtils.isNotBlank(contextID.getContextId())) {
+ contextIDKey = contextID.getContextId();
+ }
+
+ ServiceInstance mainInstance = DataWorkCloudApplication.getServiceInstance();
+ if (null == mainInstance || StringUtils.isBlank(mainInstance.getInstance())) {
+ logger.error("MainInstance cannot be null.");
+ throw new CSErrorException(ErrorCode.INVALID_INSTANCE, "MainInstance backupInstance cannot be null.");
+ }
+ String mainInstanceAlias = instanceAliasConverter.instanceToAlias(mainInstance.getInstance());
+ String backupInstance = backupInstanceGenerator.chooseBackupInstance(mainInstanceAlias);
+ if (StringUtils.isBlank(backupInstance)) {
+ logger.error("Generate backupInstance cannot be null.");
+ throw new CSErrorException(ErrorCode.GENERATE_BACKUP_INSTANCE_ERROR, "Generate backupInstance cannot be null.");
+ }
+ HAContextID haContextID = new CommonHAContextID(mainInstanceAlias, backupInstance, contextIDKey);
+ logger.info("Generate a haContextID : {}" + new Gson().toJson(haContextID));
+ return haContextID;
+ }
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/pluggable/HAContextPersistenceManagerImpl.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/pluggable/HAContextPersistenceManagerImpl.java
new file mode 100644
index 0000000..1e5f0fa
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/pluggable/HAContextPersistenceManagerImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.pluggable;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.highavailable.AbstractContextHAManager;
+import com.webank.wedatasphere.linkis.cs.persistence.ContextPersistenceManager;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.*;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/22
+ */
+@Component
+public class HAContextPersistenceManagerImpl implements ContextPersistenceManager {
+
+ @Autowired
+ private ContextIDPersistence contextIDPersistence;
+ @Autowired
+ private ContextMapPersistence contextMapPersistence;
+
+ @Autowired
+ private ContextMetricsPersistence contextMetricsPersistence;
+ @Autowired
+ private ContextIDListenerPersistence contextIDListenerPersistence;
+ @Autowired
+ private ContextKeyListenerPersistence contextKeyListenerPersistence;
+ @Autowired
+ private TransactionManager transactionManager;
+
+
+ @Autowired
+ private AbstractContextHAManager contextHAManager;
+
+ @PostConstruct
+ void init() throws CSErrorException {
+ contextIDPersistence = contextHAManager.getContextHAProxy(contextIDPersistence);
+ contextMapPersistence = contextHAManager.getContextHAProxy(contextMapPersistence);
+ contextMetricsPersistence = contextHAManager.getContextHAProxy(contextMetricsPersistence);
+ contextIDListenerPersistence = contextHAManager.getContextHAProxy(contextIDListenerPersistence);
+ contextKeyListenerPersistence = contextHAManager.getContextHAProxy(contextKeyListenerPersistence);
+ }
+
+ @Override
+ public ContextIDPersistence getContextIDPersistence() {
+ return this.contextIDPersistence;
+ }
+
+ @Override
+ public ContextMapPersistence getContextMapPersistence() {
+ return this.contextMapPersistence;
+ }
+
+
+
+
+
+ @Override
+ public ContextMetricsPersistence getContextMetricsPersistence() {
+ return this.contextMetricsPersistence;
+ }
+
+ @Override
+ public ContextIDListenerPersistence getContextIDListenerPersistence() {
+ return this.contextIDListenerPersistence;
+ }
+
+ @Override
+ public ContextKeyListenerPersistence getContextKeyListenerPersistence() {
+ return this.contextKeyListenerPersistence;
+ }
+
+ @Override
+ public TransactionManager getTransactionManager() {
+ return this.transactionManager;
+ }
+
+}
diff --git a/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/proxy/MethodInterceptorImpl.java b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/proxy/MethodInterceptorImpl.java
new file mode 100644
index 0000000..03e0a7f
--- /dev/null
+++ b/contextservice/cs-highavailable/src/main/java/com/webank/wedatasphere/linkis/cs/highavailable/proxy/MethodInterceptorImpl.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.proxy;
+
+import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.highavailable.AbstractContextHAManager;
+import com.webank.wedatasphere.linkis.cs.highavailable.exception.ErrorCode;
+import net.sf.cglib.proxy.MethodInterceptor;
+import net.sf.cglib.proxy.MethodProxy;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 基于CGLib库实现的动态代理拦截器，拦截被代理方法的参数，在被代理方法之前和之后进行增强
+ * 被代理方法调用前增强场景较多，一般对参数为HAContextID实例、参数名包含contextid的方法参数进行转换
+ * 被代理方法调用后，目前只用于将HAContextID的数字型contextID转换为HAIDKey
+ * @Author alexyang
+ * @Date 2020/2/25
+ */
+public class MethodInterceptorImpl implements MethodInterceptor {
+
+ private final static Logger logger = LoggerFactory.getLogger(MethodInterceptorImpl.class);
+ private final static Gson gson = new Gson();
+ private AbstractContextHAManager contextHAManager;
+ private Object object;
+ private final Map<Integer, String> contextIDCacheMap = new HashMap<Integer, String>();
+
+ public MethodInterceptorImpl(AbstractContextHAManager contextHAManager, Object object) {
+ this.contextHAManager = contextHAManager;
+ this.object = object;
+ }
+
+ @Override
+ public Object intercept(Object o, Method method, Object[] args, MethodProxy methodProxy) throws Throwable {
+
+ this.contextIDCacheMap.clear();
+ // 1，执行方法前转换
+ for (int i = 0; i < args.length; i++) {
+
+ // ①参数含有ContextID实例
+ if (ContextID.class.isInstance(args[i])) {
+ ContextID contextID = (ContextID)args[i];
+ convertContextIDBeforeInvoke(contextID, i);
+ }
+
+ // ②参数含有getContextID方法的
+ convertGetContextIDBeforeInvoke(args[i]);
+ }
+
+ // ③方法名含有ContextID，并且有String类型参数，取第一个转换
+ if (method.getName().toLowerCase().contains("contextid")) {
+ for (int j = 0; j < args.length; j++) {
+ if (String.class.isInstance(args[j])) {
+ String pStr = (String)args[j];
+ if (StringUtils.isNotBlank(pStr) && !StringUtils.isNumeric(pStr)) {
+ if (this.contextHAManager.getContextHAChecker().isHAIDValid(pStr)) {
+ String contextID = this.contextHAManager.getContextHAChecker().parseHAIDFromKey(pStr).getContextId();
+ args[j] = contextID;
+ } else {
+ logger.error("Invalid HAID : " + pStr + " in method : " + method.getName());
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "Invalid HAID : " + pStr + " in method : " + method.getName());
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ // 2，执行原方法
+ Object oriResult = method.invoke(this.object, args);
+
+ // 3，执行方法后处理
+ // （1）返回值处理
+ if (null != oriResult) {
+ // ①判断集合中元素是否有getContextID方法
+ if (List.class.isInstance(oriResult)) {
+ List objList = (List)oriResult;
+ for (Object oneParameter : objList) {
+ convertGetContextIDAfterInvoke(oneParameter);
+ }
+ // ②判断ContextID
+ } else if (HAContextID.class.isInstance(oriResult)) {
+ HAContextID haContextID = (HAContextID)oriResult;
+ if (StringUtils.isNumeric(haContextID.getContextId())) {
+ String haId = this.contextHAManager.getContextHAChecker().convertHAIDToHAKey(haContextID);
+ haContextID.setContextId(haId);
+ }
+ } else {
+ // ③ 返回值方法含有getContextID
+ convertGetContextIDAfterInvoke(oriResult);
+ }
+ }
+
+ // （2）请求参数还原
+ // ①参数有ContextID实例
+ for (int k = 0; k < args.length; k++) {
+ if (ContextID.class.isInstance(args[k])) {
+ if (this.contextIDCacheMap.containsKey(k)) {
+ ContextID contextID = (ContextID)args[k];
+ contextID.setContextId(this.contextIDCacheMap.get(k));
+ } else {
+ if (HAContextID.class.isInstance(args[k])) {
+ HAContextID haContextID = (HAContextID)args[k];
+ if (StringUtils.isNumeric(haContextID.getContextId())) {
+ if (StringUtils.isNotBlank(haContextID.getInstance()) && StringUtils.isNotBlank(haContextID.getBackupInstance())) {
+ String haId = this.contextHAManager.getContextHAChecker().convertHAIDToHAKey(haContextID);
+ haContextID.setContextId(haId);
+ } else {
+ logger.error("Invalid HAContextID : " + gson.toJson(haContextID));
+ throw new CSErrorException(ErrorCode.INVAID_HA_CONTEXTID, "Invalid HAContextID : " + gson.toJson(haContextID));
+ }
+ }
+ }
+ }
+ } else {
+ // ②参数含有getContextID方法
+ convertGetContextIDAfterInvoke(args[k]);
+ }
+ }
+ // ③方法名含有ContextID，并且有String类型参数 引用不需要作处理
+
+ return oriResult;
+ }
+
+
+ private void convertContextIDBeforeInvoke(ContextID contextID, int index) throws CSErrorException {
+ if (null == contextID) {
+ return;
+ }
+ if (StringUtils.isNumeric(contextID.getContextId())) {
+ if (HAContextID.class.isInstance(contextID)) {
+ logger.error("ContextId of HAContextID instance cannot be numberic. contextId : " + gson.toJson(contextID));
+ throw new CSErrorException(ErrorCode.INVALID_CONTEXTID, "ContextId of HAContextID instance cannot be numberic. contextId : " + gson.toJson(contextID));
+ }
+ } else {
+ if (HAContextID.class.isInstance(contextID)) {
+ if (null == contextID.getContextId()) {
+ this.contextHAManager.convertProxyHAID((HAContextID) contextID);
+ } else if (this.contextHAManager.getContextHAChecker().isHAIDValid(contextID.getContextId())) {
+ if (index > 0) {
+ this.contextIDCacheMap.put(index, contextID.getContextId());
+ }
+ this.contextHAManager.convertProxyHAID((HAContextID) contextID);
+ } else {
+ logger.error("Invalid haContextId. contextId : " + gson.toJson(contextID));
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "Invalid haContextId. contextId : " + gson.toJson(contextID));
+ }
+ }
+ }
+ }
+
+ private void convertGetContextIDBeforeInvoke(Object object) throws CSErrorException {
+ for (Method innerMethod : object.getClass().getMethods()) {
+ if (innerMethod.getName().toLowerCase().contains("getcontextid")) {
+ try {
+ Object result = innerMethod.invoke(object);
+ if (null != object && ContextID.class.isInstance(result)) {
+ convertContextIDBeforeInvoke((ContextID)result, -1);
+ } else {
+ logger.warn("Method {} returns non-contextid object : {}", innerMethod.getName(), gson.toJson(object));
+ }
+ } catch (Exception e) {
+ logger.error("call method : {} error, ", innerMethod.getName(), e);
+ }
+ }
+ }
+ }
+
+ private void convertGetContextIDAfterInvoke(Object object) throws CSErrorException {
+ for (Method innerMethod : object.getClass().getMethods()) {
+ convertGetContextIDAfterInvokeMethod(innerMethod);
+ }
+ }
+
+ private void convertGetContextIDAfterInvokeMethod(Method method) throws CSErrorException {
+ if (method.getName().toLowerCase().contains("getcontextid")) {
+ Object result = null;
+ try {
+ result = method.invoke(object);
+ } catch (Exception e) {
+ logger.warn("Invoke method : {} error. ", method.getName(), e);
+ }
+ if (null != result && HAContextID.class.isInstance(result)) {
+ HAContextID haContextID = (HAContextID)result;
+ if (StringUtils.isNumeric(haContextID.getContextId())
+ && StringUtils.isNotBlank(haContextID.getInstance())
+ && StringUtils.isNotBlank(haContextID.getBackupInstance())) {
+ String haid = this.contextHAManager.getContextHAChecker().convertHAIDToHAKey(haContextID);
+ haContextID.setContextId(haid);
+ } else {
+ logger.error("GetContextID method : " + method.getName() + " returns invalid haContextID : " + gson.toJson(result));
+ throw new CSErrorException(ErrorCode.INVALID_HAID, "GetContextID method : " + method.getName() + " returns invalid haContextID : " + gson.toJson(result));
+ }
+ }
+ }
+ }
+}
diff --git a/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/TestContextHAManager.java b/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/TestContextHAManager.java
new file mode 100644
index 0000000..e0ae1eb
--- /dev/null
+++ b/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/TestContextHAManager.java
@@ -0,0 +1,205 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.test;
+
+import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication;
+import com.webank.wedatasphere.linkis.common.ServiceInstance;
+import com.webank.wedatasphere.linkis.common.conf.BDPConfiguration;
+import com.webank.wedatasphere.linkis.common.conf.Configuration;
+import com.webank.wedatasphere.linkis.common.exception.DWCException;
+import com.webank.wedatasphere.linkis.common.utils.Utils;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.highavailable.AbstractContextHAManager;
+import com.webank.wedatasphere.linkis.cs.highavailable.test.haid.TestHAID;
+import com.webank.wedatasphere.linkis.cs.highavailable.test.persist.TestPersistence;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.context.event.ApplicationPreparedEvent;
+import org.springframework.boot.web.embedded.jetty.JettyServerCustomizer;
+import org.springframework.boot.web.embedded.jetty.JettyServletWebServerFactory;
+import org.springframework.boot.web.server.WebServerFactoryCustomizer;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
+import org.springframework.cloud.context.scope.refresh.RefreshScopeRefreshedEvent;
+import org.springframework.context.ApplicationListener;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.core.env.CompositePropertySource;
+import org.springframework.core.env.Environment;
+import org.springframework.core.env.PropertySource;
+import org.springframework.core.env.StandardEnvironment;
+import org.springframework.web.filter.CharacterEncodingFilter;
+
+import javax.servlet.DispatcherType;
+import java.util.EnumSet;
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/22
+ */
+@SpringBootApplication
+@EnableDiscoveryClient
+@ComponentScan(basePackages = "com.webank.wedatasphere.linkis")
+public class TestContextHAManager extends SpringBootServletInitializer {
+
+ private static ConfigurableApplicationContext applicationContext;
+ private static ServiceInstance serviceInstance;
+ private static final Gson gson = new Gson();
+
+ public static void main(String [] args) throws ReflectiveOperationException {
+
+ final SpringApplication application = new SpringApplication(TestContextHAManager.class);
+ application.addListeners(new ApplicationListener<ApplicationPreparedEvent>(){
+ public void onApplicationEvent(ApplicationPreparedEvent applicationPreparedEvent) {
+ System.out.println("add config from config server...");
+ if(applicationContext == null) {
+ applicationContext = applicationPreparedEvent.getApplicationContext();
+ }
+ System.out.println("initialize DataWorkCloud spring application...");
+ initDWCApplication();
+
+ }
+ });
+ application.addListeners(new ApplicationListener<RefreshScopeRefreshedEvent>() {
+ public void onApplicationEvent(RefreshScopeRefreshedEvent applicationEvent) {
+ System.out.println("refresh config from config server...");
+ updateRemoteConfig();
+ }
+ });
+ String listeners = ServerConfiguration.BDP_SERVER_SPRING_APPLICATION_LISTENERS().getValue();
+ if(StringUtils.isNotBlank(listeners)) {
+ for (String listener : listeners.split(",")) {
+ application.addListeners((ApplicationListener<?>) Class.forName(listener).newInstance());
+ }
+ }
+ applicationContext = application.run(args);
+
+ try {
+// Thread.sleep(3000l);
+ AbstractContextHAManager haManager = (AbstractContextHAManager) applicationContext.getBean(AbstractContextHAManager.class);
+ if (null == haManager) {
+ System.err.println("Null haManager!");
+ return ;
+ }
+ testHAManager(haManager);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ private static void initDWCApplication() {
+ serviceInstance = new ServiceInstance();
+ serviceInstance.setApplicationName(applicationContext.getEnvironment().getProperty("spring.application.name"));
+ serviceInstance.setInstance(Utils.getComputerName() + ":" + applicationContext.getEnvironment().getProperty("server.port"));
+ DWCException.setApplicationName(serviceInstance.getApplicationName());
+ DWCException.setHostname(Utils.getComputerName());
+ DWCException.setHostPort(Integer.parseInt(applicationContext.getEnvironment().getProperty("server.port")));
+ }
+
+ public static void updateRemoteConfig() {
+ addOrUpdateRemoteConfig(applicationContext.getEnvironment(), true);
+ }
+
+ public static void addRemoteConfig() {
+ addOrUpdateRemoteConfig(applicationContext.getEnvironment(), false);
+ }
+
+ private static void addOrUpdateRemoteConfig(Environment env, boolean isUpdateOrNot) {
+ StandardEnvironment environment = (StandardEnvironment) env;
+ PropertySource propertySource = environment.getPropertySources().get("bootstrapProperties");
+ if(propertySource == null) {
+ return;
+ }
+ CompositePropertySource source = (CompositePropertySource) propertySource;
+ for (String key: source.getPropertyNames()) {
+ Object val = source.getProperty(key);
+ if(val == null) {
+ continue;
+ }
+ if(isUpdateOrNot) {
+ System.out.println("update remote config => " + key + " = " + source.getProperty(key));
+ BDPConfiguration.set(key, val.toString());
+ } else {
+ System.out.println("add remote config => " + key + " = " + source.getProperty(key));
+ BDPConfiguration.setIfNotExists(key, val.toString());
+ }
+ }
+ }
+
+ @Override
+ protected SpringApplicationBuilder configure(SpringApplicationBuilder builder) {
+ return builder.sources(DataWorkCloudApplication.class);
+ }
+
+ @Bean
+ public WebServerFactoryCustomizer<JettyServletWebServerFactory> jettyFactoryCustomizer() {
+ return new WebServerFactoryCustomizer<JettyServletWebServerFactory>() {
+ public void customize(JettyServletWebServerFactory jettyServletWebServerFactory) {
+ jettyServletWebServerFactory.addServerCustomizers(new JettyServerCustomizer() {
+ public void customize(Server server) {
+ Handler[] childHandlersByClass = server.getChildHandlersByClass(WebAppContext.class);
+ final WebAppContext webApp = (WebAppContext) childHandlersByClass[0];
+ FilterHolder filterHolder = new FilterHolder(CharacterEncodingFilter.class);
+ filterHolder.setInitParameter("encoding", Configuration.BDP_ENCODING().getValue());
+ filterHolder.setInitParameter("forceEncoding", "true");
+ webApp.addFilter(filterHolder, "/*", EnumSet.allOf(DispatcherType.class));
+ BDPJettyServerHelper.setupRestApiContextHandler(webApp);
+ if(ServerConfiguration.BDP_SERVER_SOCKET_MODE().getValue()) {
+ BDPJettyServerHelper.setupControllerServer(webApp);
+ }
+ if(!ServerConfiguration.BDP_SERVER_DISTINCT_MODE().getValue()) {
+ BDPJettyServerHelper.setupWebAppContext(webApp);
+ }
+ }
+ });
+ }
+ };
+ }
+
+
+
+ // test
+ private static void testHAManager(AbstractContextHAManager contextHAManager) {
+ // 1 test create
+ TestHAID haid = new TestHAID();
+ try {
+ TestPersistence testPersistence = contextHAManager.getContextHAProxy(new TestPersistence());
+ HAContextID haContextID = testPersistence.createHAID(haid);
+ testPersistence.passHAID(haContextID);
+ testPersistence.setContextId(haContextID.getContextId());
+ } catch (CSErrorException e) {
+ e.printStackTrace();
+ }
+ System.out.println("Test HaManager End.");
+ }
+
+ public static ServiceInstance getServiceInstance() {
+ return serviceInstance;
+ }
+
+}
diff --git a/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/haid/TestHAID.java b/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/haid/TestHAID.java
new file mode 100644
index 0000000..46cdf78
--- /dev/null
+++ b/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/haid/TestHAID.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.test.haid;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+
+public class TestHAID implements HAContextID {
+
+ private String contextId;
+ private String instance;
+ private String backupInstance;
+
+ @Override
+ public String getContextId() {
+ return contextId;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextId = contextId;
+ }
+
+ @Override
+ public String getInstance() {
+ return instance;
+ }
+
+ @Override
+ public void setInstance(String instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public String getBackupInstance() {
+ return backupInstance;
+ }
+
+ @Override
+ public void setBackupInstance(String backupInstance) {
+ this.backupInstance = backupInstance;
+ }
+}
diff --git a/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/persist/TestPersistence.java b/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/persist/TestPersistence.java
new file mode 100644
index 0000000..a0c04ed
--- /dev/null
+++ b/contextservice/cs-highavailable/src/test/java/com/webank/wedatasphere/linkis/cs/highavailable/test/persist/TestPersistence.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.highavailable.test.persist;
+
+import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+
+public class TestPersistence {
+
+ private Gson gson = new Gson();
+
+ public void testPrint() {
+ System.out.println("TestPersistence: testPrint()");
+ }
+
+ public HAContextID createHAID(HAContextID haContextID) {
+ System.out.println("TestPersistence: createHAID(), params: haContextID : " + gson.toJson(haContextID));
+ haContextID.setContextId("1");
+ return haContextID;
+ }
+
+ public HAContextID passHAID(HAContextID haContextID) {
+ System.out.println("TestPersistence: passHAID(), params: haContextID : " + gson.toJson(haContextID));
+ return haContextID;
+ }
+
+ public void setContextId(String haid) {
+ System.out.println("TestPersistence: setContextId(), : " + gson.toJson(haid));
+ }
+
+
+}
diff --git a/contextservice/cs-highavailable/src/test/resources/application.yml b/contextservice/cs-highavailable/src/test/resources/application.yml
new file mode 100644
index 0000000..d6b3e8f
--- /dev/null
+++ b/contextservice/cs-highavailable/src/test/resources/application.yml
@@ -0,0 +1,36 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+server:
+ port: 9010
+spring:
+ application:
+ name: CLOUD-CONTEXTSERVICE
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://127.0.0.1:20303/eureka/
+ registry-fetch-interval-seconds: 5
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
diff --git a/contextservice/cs-highavailable/src/test/resources/linkis.properties b/contextservice/cs-highavailable/src/test/resources/linkis.properties
new file mode 100644
index 0000000..f4d3720
--- /dev/null
+++ b/contextservice/cs-highavailable/src/test/resources/linkis.properties
@@ -0,0 +1,34 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#wds.linkis.test.mode=true
+
+wds.linkis.server.version=v1
+
+wds.linkis.ldap.proxy.url=
+wds.linkis.ldap.proxy.baseDN=
+
+wds.linkis.server.restful.uri=/
+
+wds.linkis.server.web.session.timeout=1h
+
+wds.linkis.gateway.conf.enable.proxy.user=false
+
+wds.linkis.gateway.conf.url.pass.auth=/dss/
+
+wds.linkis.gateway.admin.user=hadoop
+
+wds.linkis.gateway.conf.enable.token.auth=true
diff --git a/contextservice/cs-highavailable/src/test/resources/log4j.properties b/contextservice/cs-highavailable/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a7e6854
--- /dev/null
+++ b/contextservice/cs-highavailable/src/test/resources/log4j.properties
@@ -0,0 +1,36 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
diff --git a/contextservice/cs-highavailable/src/test/resources/log4j2.xml b/contextservice/cs-highavailable/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..ad88ea5
--- /dev/null
+++ b/contextservice/cs-highavailable/src/test/resources/log4j2.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/contextservice/cs-listener/pom.xml b/contextservice/cs-listener/pom.xml
new file mode 100644
index 0000000..18c3482
--- /dev/null
+++ b/contextservice/cs-listener/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-listener</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-common</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>0.9.4</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/CSIDListener.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/CSIDListener.java
new file mode 100644
index 0000000..7f893ba
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/CSIDListener.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener;
+
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextIDEvent;
+
+/**
+ * @author peacewong
+ * @date 2020/2/15 11:24
+ */
+public interface CSIDListener extends ContextAsyncEventListener {
+
+
+
+ void onCSIDAccess(ContextIDEvent contextIDEvent);
+
+ void onCSIDADD(ContextIDEvent contextIDEvent);
+
+ void onCSIDRemoved(ContextIDEvent contextIDEvent);
+
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/CSKeyListener.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/CSKeyListener.java
new file mode 100644
index 0000000..f380528
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/CSKeyListener.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextKeyEvent;
+
+/**
+ * @author peacewong
+ * @date 2020/2/15 11:24
+ */
+public interface CSKeyListener extends ContextAsyncEventListener {
+ @Override
+ void onEvent(Event event);
+ void onCSKeyUpdate(ContextKeyEvent contextKeyEvent);
+ void onCSKeyAccess(ContextKeyEvent contextKeyEvent);
+
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/ContextAsyncEventListener.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/ContextAsyncEventListener.java
new file mode 100644
index 0000000..610d9b7
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/ContextAsyncEventListener.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.common.listener.EventListener;
+
+/**
+ * @author peacewong
+ * @date 2020/2/22 21:45
+ */
+public interface ContextAsyncEventListener extends EventListener {
+ void onEvent(Event event);
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/ListenerBus/ContextAsyncListenerBus.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/ListenerBus/ContextAsyncListenerBus.java
new file mode 100644
index 0000000..0054eeb
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/ListenerBus/ContextAsyncListenerBus.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.ListenerBus;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus;
+import com.webank.wedatasphere.linkis.cs.listener.ContextAsyncEventListener;
+import com.webank.wedatasphere.linkis.cs.listener.conf.ContextListenerConf;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/21
+ */
+public class ContextAsyncListenerBus<L extends ContextAsyncEventListener, E extends Event> extends ListenerEventBus<L, E> {
+
+
+ private static final String NAME = "ContextAsyncListenerBus";
+
+ public ContextAsyncListenerBus() {
+ super(ContextListenerConf.WDS_CS_LISTENER_ASYN_QUEUE_CAPACITY, NAME, ContextListenerConf.WDS_CS_LISTENER_ASYN_CONSUMER_THREAD_MAX, ContextListenerConf.WDS_CS_LISTENER_ASYN_CONSUMER_THREAD_FREE_TIME_MAX);
+ }
+
+ @Override
+ public void doPostEvent(L listener, E event) {
+ listener.onEvent(event);
+ }
+
+
+ private static ContextAsyncListenerBus contextAsyncListenerBus = null;
+
+ public static ContextAsyncListenerBus getInstance() {
+ if (contextAsyncListenerBus == null) {
+ synchronized (ContextAsyncListenerBus.class) {
+ if (contextAsyncListenerBus == null) {
+ contextAsyncListenerBus = new ContextAsyncListenerBus();
+ contextAsyncListenerBus.start();
+ }
+ }
+ }
+ return contextAsyncListenerBus;
+ }
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/AbstractCallbackEngine.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/AbstractCallbackEngine.java
new file mode 100644
index 0000000..d39b7a6
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/AbstractCallbackEngine.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.callback;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/20
+ */
+public interface AbstractCallbackEngine extends CallbackEngine {
+ //todo
+ //实现事件的存储和按需消费:存储这些变化的事件，并且按需消费
+ //事件超过一定时间还没被消费，自动移除
+ //cskey被五个client注册了listener，如果有挂掉，那么必须要一个最大消费时间的机制
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/CallbackEngine.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/CallbackEngine.java
new file mode 100644
index 0000000..6bbaff3
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/CallbackEngine.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.callback;
+
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean;
+
+import java.util.ArrayList;
+import java.util.Set;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/20
+ */
+public interface CallbackEngine {
+
+ ArrayList<ContextKeyValueBean> getListenerCallback(String source);
+
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/ContextIDCallbackEngine.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/ContextIDCallbackEngine.java
new file mode 100644
index 0000000..32b2361
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/ContextIDCallbackEngine.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.callback;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ListenerDomain;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/20
+ *
+ */
+public interface ContextIDCallbackEngine extends CallbackEngine {
+ void registerClient(ListenerDomain listenerDomain);
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/ContextKeyCallbackEngine.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/ContextKeyCallbackEngine.java
new file mode 100644
index 0000000..c5e8294
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/ContextKeyCallbackEngine.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.callback;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ListenerDomain;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/20
+ */
+public interface ContextKeyCallbackEngine extends CallbackEngine {
+ void registerClient(ListenerDomain listenerDomain);
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/ContextKeyValueBean.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/ContextKeyValueBean.java
new file mode 100644
index 0000000..9ded237
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/ContextKeyValueBean.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.callback.imp;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+
+import java.util.Objects;
+
+public class ContextKeyValueBean {
+
+ private ContextKey csKey;
+ private ContextValue csValue;
+ private ContextID csID;
+
+ public ContextID getCsID() {
+ return csID;
+ }
+
+ public void setCsID(ContextID csID) {
+ this.csID = csID;
+ }
+
+ public ContextKey getCsKey() {
+ return csKey;
+ }
+
+ public void setCsKey(ContextKey csKey) {
+ this.csKey = csKey;
+ }
+
+ public ContextValue getCsValue() {
+ return csValue;
+ }
+
+ public void setCsValue(ContextValue csValue) {
+ this.csValue = csValue;
+ }
+
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(csKey, csValue);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ContextKeyValueBean csmapKey = (ContextKeyValueBean) o;
+ return Objects.equals(csKey, csmapKey.csKey) &&
+ Objects.equals(csValue, csmapKey.csValue);
+ }
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/DefaultContextIDCallbackEngine.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/DefaultContextIDCallbackEngine.java
new file mode 100644
index 0000000..3e5d3b2
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/DefaultContextIDCallbackEngine.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.callback.imp;
+
+import com.google.common.collect.HashMultimap;
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.listener.ContextIDListener;
+import com.webank.wedatasphere.linkis.cs.listener.CSIDListener;
+import com.webank.wedatasphere.linkis.cs.listener.callback.ContextIDCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType.ADD;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/20
+ */
+public class DefaultContextIDCallbackEngine implements CSIDListener, ContextIDCallbackEngine {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultContextIDCallbackEngine.class);
+ private HashMultimap<String, ContextID> registerCSIDcsClients = HashMultimap.create();//key为clientSource的instance值
+
+ private List<ContextID> removedContextIDS = new ArrayList<>();
+
+
+ @Override
+ public ArrayList<ContextKeyValueBean> getListenerCallback(String source) {
+ Set<ContextID> ContextIDSets = registerCSIDcsClients.get(source);
+ ArrayList<ContextKeyValueBean> contextKeyValueBeans = new ArrayList<>();
+ for (ContextID contextID : removedContextIDS) {
+ if (ContextIDSets.contains(contextID)) {
+ ContextKeyValueBean contextKeyValueBean = new ContextKeyValueBean();
+ contextKeyValueBean.setCsID(contextID);
+ contextKeyValueBeans.add(contextKeyValueBean);
+ }
+ }
+ return contextKeyValueBeans;
+ }
+
+
+ @Override
+ public void registerClient(ListenerDomain listenerDomain) {
+ if (listenerDomain != null && listenerDomain instanceof CommonContextIDListenerDomain) {
+ CommonContextIDListenerDomain commonContextIDListenerDomain = (CommonContextIDListenerDomain) listenerDomain;
+ String source = commonContextIDListenerDomain.getSource();
+ ContextID contextID = commonContextIDListenerDomain.getContextID();
+ if (source != null && contextID != null) {
+ synchronized (registerCSIDcsClients) {
+ registerCSIDcsClients.put(source, contextID);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void onEvent (Event event){
+ DefaultContextIDEvent defaultContextIDEvent = null;
+ if (event != null && event instanceof DefaultContextIDEvent) {
+ defaultContextIDEvent = (DefaultContextIDEvent) event;
+ }
+ if (null == defaultContextIDEvent) {
+ logger.warn("defaultContextIDEvent event 为空");
+ return;
+ }
+ switch (defaultContextIDEvent.getOperateType()) {
+ //ADD, UPDATE, DELETE, REMOVEALL, ACCESS
+ case REMOVEALL:
+ onCSIDRemoved(defaultContextIDEvent);
+ break;
+ case ADD:
+ onCSIDADD(defaultContextIDEvent);
+ break;
+ case ACCESS:
+ onCSIDAccess(defaultContextIDEvent);
+ break;
+ case UPDATE:
+ break;
+ case DELETE:
+ break;
+ default:
+ logger.info("检查defaultContextIDEvent event操作类型");
+ }
+
+ }
+
+ @Override
+ public void onCSIDAccess (ContextIDEvent contextIDEvent){
+
+ }
+
+ @Override
+ public void onCSIDADD (ContextIDEvent contextIDEvent){
+
+ }
+
+ @Override
+ public void onCSIDRemoved (ContextIDEvent contextIDEvent){
+
+ DefaultContextIDEvent defaultContextIDEvent = null;
+ if (contextIDEvent != null && contextIDEvent instanceof DefaultContextIDEvent) {
+ defaultContextIDEvent = (DefaultContextIDEvent) contextIDEvent;
+ }
+ if (null == defaultContextIDEvent) {
+ return;
+ }
+ synchronized (removedContextIDS) {
+ removedContextIDS.add(defaultContextIDEvent.getContextID());
+ }
+ }
+
+ @Override
+ public void onEventError (Event event, Throwable t){
+
+ }
+
+
+ private static DefaultContextIDCallbackEngine singleDefaultContextIDCallbackEngine = null;
+
+ private DefaultContextIDCallbackEngine() {
+
+ }
+
+ public static DefaultContextIDCallbackEngine getInstance () {
+ if (singleDefaultContextIDCallbackEngine == null) {
+ synchronized (DefaultContextIDCallbackEngine.class) {
+ if (singleDefaultContextIDCallbackEngine == null) {
+ singleDefaultContextIDCallbackEngine = new DefaultContextIDCallbackEngine();
+ DefaultContextListenerManager instanceContextListenerManager = DefaultContextListenerManager.getInstance();
+ instanceContextListenerManager.getContextAsyncListenerBus().addListener(singleDefaultContextIDCallbackEngine);
+ logger.info("add listerner singleDefaultContextIDCallbackEngine success");
+ }
+ }
+ }
+ return singleDefaultContextIDCallbackEngine;
+ }
+ }
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/DefaultContextKeyCallbackEngine.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/DefaultContextKeyCallbackEngine.java
new file mode 100644
index 0000000..ab8358b
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/callback/imp/DefaultContextKeyCallbackEngine.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.callback.imp;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multiset;
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.listener.CSKeyListener;
+import com.webank.wedatasphere.linkis.cs.listener.callback.ContextKeyCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextKeyEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/20
+ */
+public class DefaultContextKeyCallbackEngine implements CSKeyListener, ContextKeyCallbackEngine {
+ private static final Logger logger = LoggerFactory.getLogger(DefaultContextKeyCallbackEngine.class);
+
+ private HashMultimap<String, ContextID> registerCSIDcsClients = HashMultimap.create();//key为clientSource的instance值
+
+ private HashMultimap<String, ContextKeyValueBean> registerCSIDcsKeyValues = HashMultimap.create();//key为 contextId 的ID值
+
+ //注册csClient及其监听的csKeys
+ @Override
+ public void registerClient(ListenerDomain listenerDomain) {
+ if (listenerDomain != null && listenerDomain instanceof CommonContextKeyListenerDomain) {
+ CommonContextKeyListenerDomain commonContextKeyListenerDomain = (CommonContextKeyListenerDomain) listenerDomain;
+ String source = commonContextKeyListenerDomain.getSource();
+ ContextID contextID = commonContextKeyListenerDomain.getContextID();
+ ContextKey contextKey = commonContextKeyListenerDomain.getContextKey();
+ if (source != null && contextID != null) {
+ synchronized (registerCSIDcsClients) {
+ logger.info("要注册的csClient和contextId: " + source + ":" + contextID);
+ registerCSIDcsClients.put(source, contextID);
+ }
+ }
+ //针对cskey生成一个bean，cskey对应的value值目前为空
+ if (contextKey != null) {
+ ContextKeyValueBean contextKeyValueBean = new ContextKeyValueBean();
+ contextKeyValueBean.setCsKey(contextKey);
+ contextKeyValueBean.setCsID(contextID);
+ synchronized (registerCSIDcsKeyValues) {
+ logger.info("要注册的contextId: " + contextID.getContextId());
+ registerCSIDcsKeyValues.put(contextID.getContextId(), contextKeyValueBean);
+ }
+ }
+ }
+ }
+
+ //通过 source 拿到 ContextID，遍历 ContextID，返回监听的 beans
+ @Override
+ public ArrayList<ContextKeyValueBean> getListenerCallback(String source) {
+ ArrayList<ContextKeyValueBean> arrayContextKeyValueBeans = new ArrayList<>();
+ Set<ContextID> contextIDS = registerCSIDcsClients.get(source);
+ //返回所有的 ContextKeyValueBean
+ if (contextIDS.size() > 0) {
+ for (ContextID csId : contextIDS) {
+ arrayContextKeyValueBeans.addAll(registerCSIDcsKeyValues.get(csId.getContextId()));
+ }
+ }
+ return arrayContextKeyValueBeans;
+ }
+
+ @Override
+ public void onEvent(Event event) {
+ DefaultContextKeyEvent defaultContextKeyEvent = null;
+ if (event != null && event instanceof DefaultContextKeyEvent) {
+ defaultContextKeyEvent = (DefaultContextKeyEvent) event;
+ }
+ if (null == defaultContextKeyEvent) {
+ logger.info("defaultContextKeyEvent event 为空");
+ return;
+ }
+ logger.info("defaultContextKeyEvent 要更新事件的ID: " + defaultContextKeyEvent.getContextID().getContextId());
+ logger.info("defaultContextKeyEvent 要更新事件的key: " + defaultContextKeyEvent.getContextKeyValue().getContextKey().getKey());
+ logger.info("defaultContextKeyEvent 要更新的value" + defaultContextKeyEvent.getContextKeyValue().getContextValue().getValue());
+ switch (defaultContextKeyEvent.getOperateType()) {
+ case UPDATE:
+ onCSKeyUpdate(defaultContextKeyEvent);
+ break;
+ case ACCESS:
+ onCSKeyAccess(defaultContextKeyEvent);
+ break;
+ default:
+ logger.info("检查defaultContextKeyEvent event操作类型");
+ }
+ }
+
+ //更新 cskey 对应的 value 值
+ @Override
+ public void onCSKeyUpdate(ContextKeyEvent cskeyEvent) {
+ DefaultContextKeyEvent defaultContextKeyEvent = null;
+ if (cskeyEvent != null && cskeyEvent instanceof DefaultContextKeyEvent) {
+ defaultContextKeyEvent = (DefaultContextKeyEvent) cskeyEvent;
+ }
+ if (null == defaultContextKeyEvent) {
+ return;
+ }
+
+ synchronized (registerCSIDcsKeyValues) {
+ //遍历所有csid,如果csid跟事件中的相同，则取出该csid所有的bean,更新所有bean中的csvalue.
+ Set<ContextKeyValueBean> contextKeyValueBeans = registerCSIDcsKeyValues.get(defaultContextKeyEvent.getContextID().getContextId());
+ for (ContextKeyValueBean contextKeyValueBean : contextKeyValueBeans) {
+ if (contextKeyValueBean.getCsKey().getKey().equals(defaultContextKeyEvent.getContextKeyValue().getContextKey().getKey())) {
+ contextKeyValueBean.setCsValue(defaultContextKeyEvent.getContextKeyValue().getContextValue());
+ }
+ }
+ }
+ }
+
+
+ //todo
+ @Override
+ public void onCSKeyAccess(ContextKeyEvent cskeyEvent) {
+
+ }
+
+ //todo
+ @Override
+ public void onEventError(Event event, Throwable t) {
+
+ }
+
+ private static DefaultContextKeyCallbackEngine singleDefaultContextKeyCallbackEngine = null;
+
+ private DefaultContextKeyCallbackEngine() {
+ }
+
+ public static DefaultContextKeyCallbackEngine getInstance() {
+ if (singleDefaultContextKeyCallbackEngine == null) {
+ synchronized (DefaultContextKeyCallbackEngine.class) {
+ if (singleDefaultContextKeyCallbackEngine == null) {
+ singleDefaultContextKeyCallbackEngine = new DefaultContextKeyCallbackEngine();
+ DefaultContextListenerManager instanceContextListenerManager = DefaultContextListenerManager.getInstance();
+ instanceContextListenerManager.getContextAsyncListenerBus().addListener(singleDefaultContextKeyCallbackEngine);
+ logger.info("add listerner singleDefaultContextKeyCallbackEngine success");
+ }
+ }
+ }
+ return singleDefaultContextKeyCallbackEngine;
+ }
+
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/conf/ContextListenerConf.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/conf/ContextListenerConf.java
new file mode 100644
index 0000000..b6a827c
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/conf/ContextListenerConf.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.conf;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/25
+ */
+public class ContextListenerConf {
+ public final static Integer WDS_CS_LISTENER_ASYN_CONSUMER_THREAD_MAX = Integer.parseInt(CommonVars.apply("wds.linkis.cs.listener.asyn.consumer.thread.max","5").getValue());
+ public final static Long WDS_CS_LISTENER_ASYN_CONSUMER_THREAD_FREE_TIME_MAX = Long.parseLong(CommonVars.apply("wds.linkis.cs.listener.asyn.consumer.freeTime.max","5000").getValue());
+ public final static Integer WDS_CS_LISTENER_ASYN_QUEUE_CAPACITY =Integer.parseInt(CommonVars.apply("wds.linkis.cs.listener.asyn.queue.size.max","300").getValue()) ;
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/ContextIDEvent.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/ContextIDEvent.java
new file mode 100644
index 0000000..9ac95fd
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/ContextIDEvent.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.event;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * @author peacewong
+ * @date 2020/2/20 15:03
+ */
+public interface ContextIDEvent extends Event {
+
+ ContextID getContextID();
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/ContextKeyEvent.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/ContextKeyEvent.java
new file mode 100644
index 0000000..35119e4
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/ContextKeyEvent.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.event;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+
+/**
+ * @author peacewong
+ * @date 2020/2/20 15:01
+ */
+public interface ContextKeyEvent extends Event {
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/enumeration/OperateType.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/enumeration/OperateType.java
new file mode 100644
index 0000000..30f31cf
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/enumeration/OperateType.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.event.enumeration;
+
+/**
+ * @author peacewong
+ * @date 2020/2/15 21:33
+ */
+public enum OperateType {
+
+ /**
+ * 对contextKey和contextId的一些操作
+ */
+ ADD, UPDATE, DELETE, REMOVEALL, ACCESS, CREATE,REMOVE
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/impl/DefaultContextIDEvent.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/impl/DefaultContextIDEvent.java
new file mode 100644
index 0000000..dbae9bc
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/impl/DefaultContextIDEvent.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.event.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType;
+
+/**
+ * @author peacewong
+ * @date 2020/2/20 15:03
+ */
+public class DefaultContextIDEvent implements ContextIDEvent {
+
+ private ContextID contextID;
+
+ public OperateType getOperateType() {
+ return operateType;
+ }
+
+ public void setOperateType(OperateType operateType) {
+ this.operateType = operateType;
+ }
+
+ //TODO
+ private OperateType operateType;
+
+ @Override
+ public ContextID getContextID() {
+ return contextID;
+ }
+
+ public void setContextID(ContextID contextID) {
+ this.contextID = contextID;
+ }
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/impl/DefaultContextKeyEvent.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/impl/DefaultContextKeyEvent.java
new file mode 100644
index 0000000..d770e5d
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/event/impl/DefaultContextKeyEvent.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.event.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.listener.event.ContextKeyEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType;
+
+/**
+ * @author peacewong
+ * @date 2020/2/15 21:30
+ */
+public class DefaultContextKeyEvent implements ContextKeyEvent {
+
+ private ContextID contextID;
+
+ private ContextKeyValue contextKeyValue;
+
+ private ContextKeyValue oldValue;
+
+
+ private OperateType operateType;
+
+ public ContextID getContextID() {
+ return contextID;
+ }
+
+ public void setContextID(ContextID contextID) {
+ this.contextID = contextID;
+ }
+
+ public ContextKeyValue getContextKeyValue() {
+ return contextKeyValue;
+ }
+
+ public void setContextKeyValue(ContextKeyValue contextKeyValue) {
+ this.contextKeyValue = contextKeyValue;
+ }
+
+ public OperateType getOperateType() {
+ return operateType;
+ }
+
+ public void setOperateType(OperateType operateType) {
+ this.operateType = operateType;
+ }
+
+ public ContextKeyValue getOldValue() {
+ return oldValue;
+ }
+
+ public void setOldValue(ContextKeyValue oldValue) {
+ this.oldValue = oldValue;
+ }
+
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/manager/ListenerManager.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/manager/ListenerManager.java
new file mode 100644
index 0000000..8836e5a
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/manager/ListenerManager.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.manager;
+
+import com.webank.wedatasphere.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextIDCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextKeyCallbackEngine;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/21
+ */
+public interface ListenerManager {
+ public ContextAsyncListenerBus getContextAsyncListenerBus(); //单例
+ public DefaultContextIDCallbackEngine getContextIDCallbackEngine(); //单例
+ public DefaultContextKeyCallbackEngine getContextKeyCallbackEngine(); //单例
+}
diff --git a/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/manager/imp/DefaultContextListenerManager.java b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/manager/imp/DefaultContextListenerManager.java
new file mode 100644
index 0000000..c9102da
--- /dev/null
+++ b/contextservice/cs-listener/src/main/java/com/webank/wedatasphere/linkis/cs/listener/manager/imp/DefaultContextListenerManager.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.manager.imp;
+
+import com.webank.wedatasphere.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextIDCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextKeyCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.manager.ListenerManager;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/21
+ */
+public class DefaultContextListenerManager implements ListenerManager {
+ @Override
+ public ContextAsyncListenerBus getContextAsyncListenerBus() {
+ ContextAsyncListenerBus contextAsyncListenerBus = ContextAsyncListenerBus.getInstance();
+ return contextAsyncListenerBus;
+ }
+
+ @Override
+ public DefaultContextIDCallbackEngine getContextIDCallbackEngine() {
+ DefaultContextIDCallbackEngine instanceIdCallbackEngine = DefaultContextIDCallbackEngine.getInstance();
+ return instanceIdCallbackEngine;
+ }
+
+ @Override
+ public DefaultContextKeyCallbackEngine getContextKeyCallbackEngine() {
+ DefaultContextKeyCallbackEngine instanceKeyCallbackEngine = DefaultContextKeyCallbackEngine.getInstance();
+ return instanceKeyCallbackEngine;
+ }
+
+ private static DefaultContextListenerManager singleDefaultContextListenerManager = null;
+
+ private DefaultContextListenerManager() {
+ }
+
+ public static DefaultContextListenerManager getInstance() {
+ if (singleDefaultContextListenerManager == null) {
+ synchronized (DefaultContextListenerManager.class) {
+ if (singleDefaultContextListenerManager == null) {
+ singleDefaultContextListenerManager = new DefaultContextListenerManager();
+ }
+ }
+ }
+ return singleDefaultContextListenerManager;
+ }
+}
diff --git a/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextID.java b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextID.java
new file mode 100644
index 0000000..3d81b45
--- /dev/null
+++ b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextID.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.test;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 20:41
+ */
+public class TestContextID implements ContextID {
+
+ String contextID;
+
+ @Override
+ public String getContextId() {
+ return contextID;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextID = contextId;
+ }
+}
diff --git a/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextKey.java b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextKey.java
new file mode 100644
index 0000000..6bd442e
--- /dev/null
+++ b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextKey.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.test;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/22
+ */
+public class TestContextKey implements ContextKey {
+ private String key;
+ private ContextType contextType;
+ @Override
+ public String getKey() {
+ return this.key;
+ }
+
+ @Override
+ public void setKey(String key) {
+ this.key=key;
+ }
+
+ @Override
+ public ContextType getContextType() {
+ return this.contextType;
+ }
+
+ @Override
+ public void setContextType(ContextType contextType) {
+ this.contextType=contextType;
+ }
+
+ @Override
+ public ContextScope getContextScope() {
+ return null;
+ }
+
+ @Override
+ public void setContextScope(ContextScope contextScope) {
+
+ }
+
+ @Override
+ public String getKeywords() {
+ return null;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+
+ }
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+ @Override
+ public void setType(int type) {
+
+ }
+}
diff --git a/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextKeyValue.java b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextKeyValue.java
new file mode 100644
index 0000000..df44a38
--- /dev/null
+++ b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextKeyValue.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.test;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+
+/**
+ * @author chaogefeng
+ * @date 2020/2/22 16:46
+ */
+public class TestContextKeyValue implements ContextKeyValue {
+
+ private ContextKey contextKey;
+
+ private ContextValue contextValue;
+
+ @Override
+ public ContextKey getContextKey() {
+ return this.contextKey;
+ }
+
+ @Override
+ public void setContextKey(ContextKey contextKey) {
+ this.contextKey = contextKey;
+ }
+
+ @Override
+ public ContextValue getContextValue() {
+ return this.contextValue;
+ }
+
+ @Override
+ public void setContextValue(ContextValue contextValue) {
+ this.contextValue = contextValue;
+ }
+}
diff --git a/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextValue.java b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextValue.java
new file mode 100644
index 0000000..db01d50
--- /dev/null
+++ b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestContextValue.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.test;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ValueBean;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/22
+ */
+public class TestContextValue implements ContextValue {
+ private Object value;
+
+ private String keywords;
+
+
+ @Override
+ public String getKeywords() {
+ return null;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+
+ }
+
+ @Override
+ public Object getValue() {
+ return this.value;
+ }
+
+ @Override
+ public void setValue(Object value) {
+ this.value=value;
+ }
+
+
+}
diff --git a/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestListenerManager.java b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestListenerManager.java
new file mode 100644
index 0000000..37c1740
--- /dev/null
+++ b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestListenerManager.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.listener.test;
+
+import com.webank.wedatasphere.linkis.common.listener.Event;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextIDCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextKeyCallbackEngine;
+import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextIDEvent;
+import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent;
+import com.webank.wedatasphere.linkis.cs.listener.manager.ListenerManager;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Author: chaogefeng
+ * @Date: 2020/2/22
+ */
+public class TestListenerManager {
+ @Test
+ public void testGetContextAsyncListenerBus() {
+ DefaultContextListenerManager defaultContextListenerManager = DefaultContextListenerManager.getInstance();
+
+ ContextAsyncListenerBus contextAsyncListenerBus = defaultContextListenerManager.getContextAsyncListenerBus();
+
+ DefaultContextIDCallbackEngine contextIDCallbackEngine = defaultContextListenerManager.getContextIDCallbackEngine();
+
+ DefaultContextKeyCallbackEngine contextKeyCallbackEngine = defaultContextListenerManager.getContextKeyCallbackEngine();
+ //client1的contextID
+ TestContextID testContextID1 = new TestContextID();
+ testContextID1.setContextId("18392881376");
+
+ //client2的contextID
+ TestContextID testContextID2 = new TestContextID();
+ testContextID2.setContextId("13431335441");
+
+ List<ContextKey> csKeys1 = new ArrayList<>();
+ TestContextKey testContextKey1 = new TestContextKey();
+ testContextKey1.setKey("key1");
+ TestContextKey testContextKey2 = new TestContextKey();
+ testContextKey2.setKey("key2");
+ csKeys1.add(testContextKey1);
+ csKeys1.add(testContextKey2);
+
+ List<ContextKey> csKeys2 = new ArrayList<>();
+ TestContextKey testContextKey3 = new TestContextKey();
+ testContextKey3.setKey("key3");
+ TestContextKey testContextKey4 = new TestContextKey();
+ testContextKey4.setKey("key4");
+ csKeys2.add(testContextKey3);
+ csKeys2.add(testContextKey4);
+
+
+ ListenerDomain ListenerDomain1;
+
+ ListenerDomain ListenerDomain2;
+
+ ListenerDomain ListenerDomain3;
+
+
+
+
+ DefaultContextKeyEvent defaultContextKeyEvent = new DefaultContextKeyEvent();
+ defaultContextKeyEvent.setContextID(testContextID1);
+ defaultContextKeyEvent.setOperateType(OperateType.UPDATE);
+ TestContextKeyValue testContextKeyValue = new TestContextKeyValue();
+ testContextKeyValue.setContextKey(testContextKey1);
+ TestContextValue testContextValue = new TestContextValue();
+ testContextValue.setValue("chaogefeng");
+ testContextKeyValue.setContextValue(testContextValue);
+ defaultContextKeyEvent.setContextKeyValue(testContextKeyValue);
+ contextAsyncListenerBus.doPostEvent(contextKeyCallbackEngine, defaultContextKeyEvent);
+ ArrayList<ContextKeyValueBean> clientSource2ListenerCallback = contextKeyCallbackEngine.getListenerCallback("127.0.0.1:8888");
+ System.out.println("----------------------------------------------------------------------");
+ for (ContextKeyValueBean contextKeyValueBean : clientSource2ListenerCallback) {
+ System.out.println("返回的bean里面对应的contexID: " + contextKeyValueBean.getCsID().getContextId());
+ System.out.println("返回的bean里面对应的cskeys: " + contextKeyValueBean.getCsKey().getKey());
+ if (contextKeyValueBean.getCsValue() != null) {
+ System.out.println("返回的bean里面对应的value: " + contextKeyValueBean.getCsValue().getValue());
+ }
+ }
+ }
+
+}
diff --git a/contextservice/cs-persistence/pom.xml b/contextservice/cs-persistence/pom.xml
new file mode 100644
index 0000000..2895b19
--- /dev/null
+++ b/contextservice/cs-persistence/pom.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-persistence</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-mybatis</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.1.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/java</directory>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceBeans.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceBeans.java
new file mode 100644
index 0000000..db9834b
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceBeans.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence;
+
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.*;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Created by patinousward on 2020/2/23.
+ */
+@Configuration
+public class ContextPersistenceBeans {
+
+ @Bean
+ @ConditionalOnMissingBean(ContextPersistenceManager.class)
+ public ContextPersistenceManager getContextPersistenceManager(
+ ContextIDPersistence contextIDPersistence,
+ ContextMapPersistence contextMapPersistence,
+ ContextMetricsPersistence contextMetricsPersistence,
+ ContextIDListenerPersistence contextIDListenerPersistence,
+ ContextKeyListenerPersistence contextKeyListenerPersistence,
+ TransactionManager transactionManager
+ ) {
+ ContextPersistenceManagerImpl manager = new ContextPersistenceManagerImpl();
+ manager.setContextIDPersistence(contextIDPersistence);
+ manager.setContextMapPersistence(contextMapPersistence);
+ manager.setContextMetricsPersistence(contextMetricsPersistence);
+ manager.setContextKeyListenerPersistence(contextKeyListenerPersistence);
+ manager.setContextIDListenerPersistence(contextIDListenerPersistence);
+ manager.setTransactionManager(transactionManager);
+ return manager;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceManager.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceManager.java
new file mode 100644
index 0000000..90bf6d7
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceManager.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence;
+
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.*;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextPersistenceManager {
+
+ ContextIDPersistence getContextIDPersistence();
+
+ ContextMapPersistence getContextMapPersistence();
+
+
+
+ ContextMetricsPersistence getContextMetricsPersistence();
+
+ ContextIDListenerPersistence getContextIDListenerPersistence();
+
+ ContextKeyListenerPersistence getContextKeyListenerPersistence();
+
+ TransactionManager getTransactionManager();
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceManagerImpl.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceManagerImpl.java
new file mode 100644
index 0000000..a0fdfbf
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/ContextPersistenceManagerImpl.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence;
+
+import com.webank.wedatasphere.linkis.cs.persistence.annotation.Tuning;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.*;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+public class ContextPersistenceManagerImpl implements ContextPersistenceManager {
+
+
+ private ContextIDPersistence contextIDPersistence;
+
+ private ContextMapPersistence contextMapPersistence;
+
+
+
+ private ContextMetricsPersistence contextMetricsPersistence;
+
+ private ContextIDListenerPersistence contextIDListenerPersistence;
+
+ private ContextKeyListenerPersistence contextKeyListenerPersistence;
+
+ private TransactionManager transactionManager;
+
+ @Override
+ @Tuning
+ public ContextIDPersistence getContextIDPersistence() {
+ return this.contextIDPersistence;
+ }
+
+ @Override
+ @Tuning
+ public ContextMapPersistence getContextMapPersistence() {
+ return this.contextMapPersistence;
+ }
+
+
+
+ @Override
+ @Tuning
+ public ContextMetricsPersistence getContextMetricsPersistence() {
+ return this.contextMetricsPersistence;
+ }
+
+ @Override
+ @Tuning
+ public ContextIDListenerPersistence getContextIDListenerPersistence() {
+ return this.contextIDListenerPersistence;
+ }
+
+ @Override
+ @Tuning
+ public ContextKeyListenerPersistence getContextKeyListenerPersistence() {
+ return this.contextKeyListenerPersistence;
+ }
+
+ @Override
+ public TransactionManager getTransactionManager() {
+ return this.transactionManager;
+ }
+
+ public void setContextIDPersistence(ContextIDPersistence contextIDPersistence) {
+ this.contextIDPersistence = contextIDPersistence;
+ }
+
+ public void setContextMapPersistence(ContextMapPersistence contextMapPersistence) {
+ this.contextMapPersistence = contextMapPersistence;
+ }
+
+
+ public void setContextMetricsPersistence(ContextMetricsPersistence contextMetricsPersistence) {
+ this.contextMetricsPersistence = contextMetricsPersistence;
+ }
+
+ public void setContextIDListenerPersistence(ContextIDListenerPersistence contextIDListenerPersistence) {
+ this.contextIDListenerPersistence = contextIDListenerPersistence;
+ }
+
+ public void setContextKeyListenerPersistence(ContextKeyListenerPersistence contextKeyListenerPersistence) {
+ this.contextKeyListenerPersistence = contextKeyListenerPersistence;
+ }
+
+ public void setTransactionManager(TransactionManager transactionManager) {
+ this.transactionManager = transactionManager;
+ }
+
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/annotation/Ignore.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/annotation/Ignore.java
new file mode 100644
index 0000000..b92ec0c
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/annotation/Ignore.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Created by patinousward on 2020/2/14.
+ */
+@Target({ElementType.TYPE, ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Ignore {
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/annotation/Tuning.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/annotation/Tuning.java
new file mode 100644
index 0000000..d680b91
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/annotation/Tuning.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Created by patinousward on 2020/2/23.
+ */
+@Target({ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Tuning {
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/aop/PersistenceTuningAspect.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/aop/PersistenceTuningAspect.java
new file mode 100644
index 0000000..82edc2f
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/aop/PersistenceTuningAspect.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.aop;
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication;
+import com.webank.wedatasphere.linkis.cs.persistence.conf.PersistenceConf;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.Signature;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.aspectj.lang.reflect.MethodSignature;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.stereotype.Component;
+
+import java.lang.reflect.Method;
+
+/**
+ * Created by patinousward on 2020/2/23.
+ */
+@Aspect
+@Component
+public class PersistenceTuningAspect {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private boolean tuningIsOpen = false;
+
+ private Object tuningObject = null;
+
+ private Method tuningMethod = null;
+
+ private boolean isInited = false;
+
+ public void init() {
+ synchronized (this) {
+ if (!isInited) {
+ try {
+ Class<?> tuningClass = this.getClass().getClassLoader().
+ loadClass(PersistenceConf.TUNING_CLASS.getValue());
+ ApplicationContext context = DataWorkCloudApplication.getApplicationContext();
+ if (context != null) {
+ try {
+ tuningObject = context.getBean(tuningClass);
+ logger.info("find singleton tuning Object from IOC");
+ } catch (NoSuchBeanDefinitionException e) {
+ logger.info("can not find singleton tuning Object from IOC");
+ }
+ }
+ if (tuningObject == null) {
+ tuningObject = tuningClass.newInstance();
+ }
+ tuningMethod = tuningClass.getMethod(PersistenceConf.TUNING_METHOD.getValue(), Object.class);
+ tuningIsOpen = true;
+ } catch (ClassNotFoundException | InstantiationException |
+ IllegalAccessException | NoSuchMethodException e) {
+ logger.warn("can not load tuning class,tuning is close", e);
+ } finally {
+ isInited = true;
+ }
+ }
+ }
+ }
+
+ @Pointcut(value = "@annotation(com.webank.wedatasphere.linkis.cs.persistence.annotation.Tuning)")
+ private void cut() {
+ }
+
+ @Around("cut()")
+ public Object around(ProceedingJoinPoint point) throws Throwable {
+ if (!isInited) {
+ init();
+ }
+ if (!tuningIsOpen) {
+ logger.info("tuning is close..return the real");
+ return point.proceed();
+ }
+ Signature signature = point.getSignature();
+ if (!(signature instanceof MethodSignature)) {
+ throw new IllegalArgumentException("该注解只能用于方法");
+ }
+ MethodSignature methodSignature = (MethodSignature) signature;
+ Object target = point.getTarget();
+ Method currentMethod = target.getClass().getMethod(methodSignature.getName(), methodSignature.getParameterTypes());
+ logger.info("调用方法：" + currentMethod.getName());
+ return tuningMethod.invoke(tuningObject, point.proceed());
+ }
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/conf/PersistenceConf.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/conf/PersistenceConf.java
new file mode 100644
index 0000000..5df07a4
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/conf/PersistenceConf.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.conf;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * Created by patinousward on 2020/2/13.
+ */
+public class PersistenceConf {
+
+ public static final CommonVars<String> TUNING_CLASS = CommonVars.apply("wds.linkis.cs.ha.class", "com.webank.wedatasphere.linkis.cs.highavailable.DefaultContextHAManager");
+ //public static final CommonVars<String> TUNING_CLASS = CommonVars.apply("wds.linkis.cs.ha.class","com.webank.wedatasphere.linkis.cs.persistence.ProxyMethodA");
+
+ public static final CommonVars<String> TUNING_METHOD = CommonVars.apply("wds.linkis.cs.ha.proxymethod", "getContextHAProxy");
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextIDListenerMapper.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextIDListenerMapper.java
new file mode 100644
index 0000000..0492a5a
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextIDListenerMapper.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.dao;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextIDListener;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+public interface ContextIDListenerMapper {
+
+ void createIDListener(@Param("listener") PersistenceContextIDListener listener);
+
+ void remove(@Param("listener") PersistenceContextIDListener listener);
+
+ void removeAll(@Param("contextID") ContextID contextID);
+
+ List<PersistenceContextIDListener> getAll(@Param("contextID") ContextID contextID);
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextIDMapper.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextIDMapper.java
new file mode 100644
index 0000000..7178641
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextIDMapper.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.dao;
+
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextID;
+
+/**
+ * Created by patinousward on 2020/2/13.
+ */
+public interface ContextIDMapper {
+ void createContextID(PersistenceContextID persistenceContextID);
+
+ void deleteContextID(String contextId);
+
+ PersistenceContextID getContextID(String contextId);
+
+ void updateContextID(PersistenceContextID persistenceContextID);
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextKeyListenerMapper.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextKeyListenerMapper.java
new file mode 100644
index 0000000..979e524
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextKeyListenerMapper.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.dao;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyListener;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+public interface ContextKeyListenerMapper {
+
+ void createKeyListener(@Param("listener") PersistenceContextKeyListener listener);
+
+ void remove(@Param("listener") ContextKeyListenerDomain contextKeyListenerDomain, @Param("keyId") Integer keyId);
+
+ void removeAll(@Param("keyIds") List<Integer> keyIds);
+
+ List<PersistenceContextKeyListener> getAll(@Param("keyIds") List<Integer> keyIds);
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextMapMapper.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextMapMapper.java
new file mode 100644
index 0000000..422b2ce
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/ContextMapMapper.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.dao;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyValue;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/13.
+ */
+public interface ContextMapMapper {
+ void createMap(PersistenceContextKeyValue pKV);
+
+ void updateMap(PersistenceContextKeyValue pKV);
+
+ PersistenceContextKeyValue getContextMap(@Param("contextID") ContextID contextID, @Param("contextKey") ContextKey contextKey);
+
+ List<PersistenceContextKeyValue> getAllContextMapByKey(@Param("contextID") ContextID contextID, @Param("key") String key);
+
+ List<PersistenceContextKeyValue> getAllContextMapByContextID(@Param("contextID") ContextID contextID);
+
+ List<PersistenceContextKeyValue> getAllContextMapByScope(@Param("contextID") ContextID contextID, @Param("contextScope") ContextScope contextScope);
+
+ List<PersistenceContextKeyValue> getAllContextMapByType(@Param("contextID") ContextID contextID, @Param("contextType") ContextType contextType);
+
+ void removeContextMap(@Param("contextID") ContextID contextID, @Param("contextKey") ContextKey contextKey);
+
+ void removeAllContextMapByContextID(@Param("contextID") ContextID contextID);
+
+ void removeAllContextMapByType(@Param("contextID") ContextID contextID, @Param("contextType") ContextType contextType);
+
+ void removeAllContextMapByScope(@Param("contextID") ContextID contextID, @Param("contextScope") ContextScope contextScope);
+
+ void removeByKeyPrefixAndContextType(@Param("contextID") ContextID contextID, @Param("contextType") ContextType contextType, @Param("keyPrefix") String keyPrefix);
+
+ void removeByKeyPrefix(@Param("contextID") ContextID contextID, @Param("keyPrefix") String keyPrefix);
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextIDListenerMapper.xml b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextIDListenerMapper.xml
new file mode 100644
index 0000000..f4a6fc7
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextIDListenerMapper.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+
+<mapper namespace="com.webank.wedatasphere.linkis.cs.persistence.dao.ContextIDListenerMapper">
+
+ <sql id="context_id_listener">
+ `id`,`listener_source`,`context_id`
+ </sql>
+
+ <insert id="createIDListener" useGeneratedKeys="true" keyProperty="id"
+ parameterType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextIDListener">
+ INSERT INTO linkis_cs_context_listener(<include refid="context_id_listener"/>)
+ VALUES (#{listener.id},#{listener.source},#{listener.contextId})
+ </insert>
+
+ <delete id="remove">
+ DELETE FROM linkis_cs_context_listener where context_id = #{listener.contextId} AND listener_source = #{listener.source}
+ </delete>
+
+ <delete id="removeAll">
+ DELETE FROM linkis_cs_context_listener where context_id = #{contextID.contextId}
+ </delete>
+
+ <select id="getAll" resultType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextIDListener">
+ SELECT * FROM linkis_cs_context_listener where context_id = #{contextID.contextId}
+ </select>
+
+
+</mapper>
\ No newline at end of file
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextIDMapper.xml b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextIDMapper.xml
new file mode 100644
index 0000000..1235e6d
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextIDMapper.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+
+<mapper namespace="com.webank.wedatasphere.linkis.cs.persistence.dao.ContextIDMapper">
+
+ <sql id="context_id">
+ `id`,`user`,`application`,`source`,`expire_type`,`expire_time`,`instance`,`backup_instance`
+ </sql>
+
+ <insert id="createContextID" useGeneratedKeys="true" keyProperty="contextId"
+ parameterType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextID">
+ INSERT INTO linkis_cs_context_id(<include refid="context_id"/>)
+ VALUES (#{contextId},#{user},#{application},#{source},#{expireType},#{expireTime},#{instance},#{backupInstance})
+ </insert>
+
+ <delete id="deleteContextID">
+ DELETE FROM linkis_cs_context_id where id = #{contextId}
+ </delete>
+
+ <resultMap id="PersistenceContextID"
+ type="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextID">
+ <id column="id" property="contextId"/>
+ </resultMap>
+
+ <select id="getContextID" resultMap="PersistenceContextID">
+ SELECT * FROM linkis_cs_context_id where id = #{contextId}
+ </select>
+
+ <update id="updateContextID"
+ parameterType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextID">
+ UPDATE linkis_cs_context_id
+ <trim prefix="set" suffixOverrides=",">
+ <if test="user != null">`user` = #{user},</if>
+ <if test="application != null">`application` = #{application},</if>
+ <if test="expireType != null">`expire_type` = #{expireType},</if>
+ <if test="expireTime != null">`expire_time` = #{expireTime},</if>
+ <if test="instance != null">`instance` = #{instance},</if>
+ <if test="backupInstance != null">`backup_instance` = #{backupInstance},</if>
+ </trim>
+ WHERE id = #{contextId}
+ </update>
+
+</mapper>
\ No newline at end of file
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextKeyListenerMapper.xml b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextKeyListenerMapper.xml
new file mode 100644
index 0000000..094b000
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextKeyListenerMapper.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+
+<mapper namespace="com.webank.wedatasphere.linkis.cs.persistence.dao.ContextKeyListenerMapper">
+
+ <sql id="context_key_listener">
+ `id`,`listener_source`,`key_id`
+ </sql>
+
+ <insert id="createKeyListener" useGeneratedKeys="true" keyProperty="id"
+ parameterType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyListener">
+ INSERT INTO linkis_cs_context_map_listener(<include refid="context_key_listener"/>)
+ VALUES (#{listener.id},#{listener.source},#{listener.keyId})
+ </insert>
+
+ <delete id="remove">
+ DELETE FROM linkis_cs_context_map_listener WHERE listener_source = #{listener.source} AND key_id = #{keyId}
+ </delete>
+
+ <delete id="removeAll">
+ DELETE FROM linkis_cs_context_map_listener WHERE
+ <foreach collection="keyIds" index="index" item="item" separator="," open="key_id in (" close=")">
+ #{item}
+ </foreach>
+ </delete>
+
+ <select id="getAll" resultType="com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextKeyListenerPersistence">
+ SELECT * FROM linkis_cs_context_map_listener WHERE
+ <foreach collection="keyIds" item="keyid" index="i" open="key_id in (" close=")" separator=",">
+ #{keyid}
+ </foreach>
+ </select>
+
+</mapper>
\ No newline at end of file
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextMapMapper.xml b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextMapMapper.xml
new file mode 100644
index 0000000..3c843b7
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/dao/impl/contextMapMapper.xml
@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+
+
+<mapper namespace="com.webank.wedatasphere.linkis.cs.persistence.dao.ContextMapMapper">
+
+ <sql id="context_map">
+ `id`,`key`,`context_scope`,`context_type`,`props`,`value`,`context_id`,`keywords`
+ </sql>
+
+ <insert id="createMap"
+ parameterType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyValue">
+ INSERT INTO linkis_cs_context_map(<include refid="context_map"/>)
+ VALUES (#{id},#{contextKey.key},#{contextKey.contextScope},#{contextKey.contextType},
+ #{props},#{contextValue.valueStr},#{contextId},#{contextValue.keywords})
+ </insert>
+
+ <resultMap id="PersistenceContextKeyValue"
+ type="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyValue">
+ <id column="id" property="id"></id>
+ <result column="context_id" property="contextId"></result>
+ <result column="props" property="props"></result>
+ <association property="contextKey"
+ javaType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKey">
+ <result column="key" property="key"></result>
+ <result column="context_scope" property="contextScope"></result>
+ <result column="context_type" property="contextType"></result>
+ <result column="keywords" property="keywords"></result>
+ </association>
+ <association property="contextValue"
+ javaType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextValue">
+ <result column="value" property="valueStr"></result>
+ </association>
+ </resultMap>
+
+ <select id="getContextMap" resultMap="PersistenceContextKeyValue">
+ SELECT * FROM linkis_cs_context_map
+ <where>
+ <if test="contextID.contextId != null">context_id = #{contextID.contextId}</if>
+ <if test="contextKey.key != null">AND `key` = #{contextKey.key}</if>
+ </where>
+ </select>
+
+ <select id="getAllContextMapByKey" resultMap="PersistenceContextKeyValue">
+ SELECT * FROM linkis_cs_context_map WHERE context_id = #{contextID.contextId} AND `key` LIKE "%"#{key}"%"
+ </select>
+
+
+ <select id="getAllContextMapByContextID" resultMap="PersistenceContextKeyValue">
+ SELECT * FROM linkis_cs_context_map WHERE context_id = #{contextID.contextId}
+ </select>
+
+ <select id="getAllContextMapByScope" resultMap="PersistenceContextKeyValue">
+ SELECT * FROM linkis_cs_context_map WHERE context_id = #{contextID.contextId} AND context_scope = #{contextScope}
+ </select>
+
+ <select id="getAllContextMapByType" resultMap="PersistenceContextKeyValue">
+ SELECT * FROM linkis_cs_context_map WHERE context_id = #{contextID.contextId} AND context_type = #{contextType}
+ </select>
+
+ <delete id="removeContextMap">
+ DELETE FROM linkis_cs_context_map
+ <where>
+ <if test="contextID.contextId != null">context_id = #{contextID.contextId}</if>
+ <if test="contextKey.key != null">AND `key` = #{contextKey.key}</if>
+ </where>
+ </delete>
+
+ <delete id="removeAllContextMapByContextID">
+ DELETE FROM linkis_cs_context_map WHERE context_id = #{contextID.contextId}
+ </delete>
+
+ <delete id="removeAllContextMapByType">
+ DELETE FROM linkis_cs_context_map WHERE context_id = #{contextID.contextId} AND context_type = #{contextType}
+ </delete>
+
+ <delete id="removeAllContextMapByScope">
+ DELETE FROM linkis_cs_context_map WHERE context_id = #{contextID.contextId} AND context_scope = #{contextScope}
+ </delete>
+
+ <update id="updateMap"
+ parameterType="com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyValue">
+ UPDATE linkis_cs_context_map
+ <trim prefix="set" suffixOverrides=",">
+ <if test="contextKey.contextScope != null">`context_scope` = #{contextKey.contextScope},</if>
+ <if test="contextKey.contextType != null">`context_type` = #{contextKey.contextType},</if>
+ <if test="contextValue.valueStr != null">`value` = #{contextValue.valueStr},</if>
+ <if test="contextValue.keywords != null">`keywords` = #{contextValue.keywords},</if>
+ </trim>
+ WHERE context_id = #{contextId} AND `key` = #{contextKey.key}
+ </update>
+
+ <delete id="removeByKeyPrefix">
+ delete from linkis_cs_context_map
+ <where>
+ <if test="#{contextID.contextId} !=null">context_id = #{contextID.contextId} and </if>
+ <!--<if test="#{contextType} !=null">context_type = #{contextType} and </if>-->
+ <if test="#{keyPrefix} !=null">`key` like concat(#{keyPrefix},'%')</if>
+ </where>
+ </delete>
+ <!--todo 和上面的合并sql-->
+ <delete id="removeByKeyPrefixAndContextType">
+ delete from linkis_cs_context_map
+ <where>
+ <if test="#{contextID.contextId} !=null">context_id = #{contextID.contextId} and </if>
+ <if test="#{contextType} !=null">context_type = #{contextType} and </if>
+ <if test="#{keyPrefix} !=null">`key` like concat(#{keyPrefix},'%')</if>
+ </where>
+ </delete>
+
+</mapper>
\ No newline at end of file
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/ExtraFieldClass.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/ExtraFieldClass.java
new file mode 100644
index 0000000..82d1223
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/ExtraFieldClass.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.entity;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/14.
+ */
+public class ExtraFieldClass {
+
+ private String className;
+
+ private List<ExtraFieldClass> subs = new ArrayList<>();
+
+ private List<String> fieldNames = new ArrayList<>();
+
+ private List<Object> fieldValues = new ArrayList<>();
+
+ private List<String> fieldTypes = new ArrayList<>();
+
+ public void addSub(ExtraFieldClass sub) {
+ subs.add(sub);
+ }
+
+ public void addFieldName(String fieldName) {
+ fieldNames.add(fieldName);
+ }
+
+ public void addFieldValue(Object fieldValue) {
+ fieldValues.add(fieldValue);
+ }
+
+ public void addFieldType(String fieldtype) {
+ fieldTypes.add(fieldtype);
+ }
+
+ public ExtraFieldClass getOneSub(int index) {
+ return subs.get(index);
+ }
+
+ public String getOneFieldName(int index) {
+ return fieldNames.get(index);
+ }
+
+ public String getOneFieldType(int index) {
+ return fieldTypes.get(index);
+ }
+
+ public Object getOneFieldValue(int index) {
+ return fieldValues.get(index);
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public void setClassName(String className) {
+ this.className = className;
+ }
+
+ public List<ExtraFieldClass> getSubs() {
+ return subs;
+ }
+
+ public void setSubs(List<ExtraFieldClass> subs) {
+ this.subs = subs;
+ }
+
+ public List<String> getFieldNames() {
+ return fieldNames;
+ }
+
+ public void setFieldNames(List<String> fieldNames) {
+ this.fieldNames = fieldNames;
+ }
+
+ public List<Object> getFieldValues() {
+ return fieldValues;
+ }
+
+ public void setFieldValues(List<Object> fieldValues) {
+ this.fieldValues = fieldValues;
+ }
+
+ public List<String> getFieldTypes() {
+ return fieldTypes;
+ }
+
+ public void setFieldTypes(List<String> fieldTypes) {
+ this.fieldTypes = fieldTypes;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextID.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextID.java
new file mode 100644
index 0000000..cd9fb37
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextID.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.entity;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ExpireType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.HAContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.UserContextID;
+import com.webank.wedatasphere.linkis.cs.persistence.annotation.Ignore;
+
+import java.util.Date;
+
+/**
+ * Created by patinousward on 2020/2/12.
+ */
+@Ignore
+public class PersistenceContextID implements UserContextID, HAContextID {
+
+ private String contextId;
+
+ private String user;
+
+ private String instance;
+
+ private String backupInstance;
+
+ private String application;
+
+ private ExpireType expireType;
+
+ private Date expireTime;
+
+ private String source;
+
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ @Override
+ public String getContextId() {
+ return this.contextId;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextId = contextId;
+ }
+
+ @Override
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ @Override
+ public String getUser() {
+ return this.user;
+ }
+
+ @Override
+ public String getInstance() {
+ return this.instance;
+ }
+
+ @Override
+ public void setInstance(String instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public String getBackupInstance() {
+ return this.backupInstance;
+ }
+
+ @Override
+ public void setBackupInstance(String backupInstance) {
+ this.backupInstance = backupInstance;
+ }
+
+ public String getApplication() {
+ return application;
+ }
+
+ public void setApplication(String application) {
+ this.application = application;
+ }
+
+ public ExpireType getExpireType() {
+ return expireType;
+ }
+
+ public void setExpireType(ExpireType expireType) {
+ this.expireType = expireType;
+ }
+
+ public Date getExpireTime() {
+ return expireTime;
+ }
+
+ public void setExpireTime(Date expireTime) {
+ this.expireTime = expireTime;
+ }
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextIDListener.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextIDListener.java
new file mode 100644
index 0000000..1dc0894
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextIDListener.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.entity;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ListenerDomain;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+public class PersistenceContextIDListener implements ListenerDomain {
+
+ private Integer id;
+
+ private String source;
+
+ private String contextId;
+
+ public Integer getId() {
+ return id;
+ }
+
+ public void setId(Integer id) {
+ this.id = id;
+ }
+
+ public String getContextId() {
+ return contextId;
+ }
+
+ public void setContextId(String contextId) {
+ this.contextId = contextId;
+ }
+
+ @Override
+ public String getSource() {
+ return this.source;
+ }
+
+ @Override
+ public void setSource(String source) {
+ this.source = source;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKey.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKey.java
new file mode 100644
index 0000000..eb5d77c
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKey.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.entity;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.persistence.annotation.Ignore;
+
+/**
+ * Created by patinousward on 2020/2/12.
+ */
+@Ignore
+public class PersistenceContextKey implements ContextKey {
+
+ private String key;
+
+ private String keywords;
+
+ private ContextScope contextScope;
+
+ private ContextType contextType;
+
+ @Override
+ public String getKey() {
+ return this.key;
+ }
+
+ @Override
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public ContextType getContextType() {
+ return this.contextType;
+ }
+
+ @Override
+ public void setContextType(ContextType contextType) {
+ this.contextType = contextType;
+ }
+
+ @Override
+ public ContextScope getContextScope() {
+ return this.contextScope;
+ }
+
+ @Override
+ public void setContextScope(ContextScope contextScope) {
+ this.contextScope = contextScope;
+ }
+
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+ @Override
+ public void setType(int type) {
+
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKeyListener.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKeyListener.java
new file mode 100644
index 0000000..9f6278d
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKeyListener.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.entity;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ListenerDomain;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+public class PersistenceContextKeyListener implements ListenerDomain {
+
+ private Integer id;
+
+ private String source;
+
+ private Integer keyId;
+
+ public Integer getId() {
+ return id;
+ }
+
+ public void setId(Integer id) {
+ this.id = id;
+ }
+
+ public Integer getKeyId() {
+ return keyId;
+ }
+
+ public void setKeyId(Integer keyId) {
+ this.keyId = keyId;
+ }
+
+ @Override
+ public String getSource() {
+ return this.source;
+ }
+
+ @Override
+ public void setSource(String source) {
+ this.source = source;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKeyValue.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKeyValue.java
new file mode 100644
index 0000000..713cc87
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextKeyValue.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.entity;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.persistence.annotation.Ignore;
+
+/**
+ * Created by patinousward on 2020/2/12.
+ */
+@Ignore
+public class PersistenceContextKeyValue implements ContextKeyValue {
+
+ private Integer id;
+
+ private String contextId;
+
+ private ContextKey contextKey;
+
+ private ContextValue contextValue;
+
+ private String props;
+
+ public String getProps() {
+ return props;
+ }
+
+ public void setProps(String props) {
+ this.props = props;
+ }
+
+ public Integer getId() {
+ return id;
+ }
+
+ public void setId(Integer id) {
+ this.id = id;
+ }
+
+ @Override
+ public void setContextKey(ContextKey contextKey) {
+ this.contextKey = contextKey;
+ }
+
+ @Override
+ public ContextKey getContextKey() {
+ return this.contextKey;
+ }
+
+ @Override
+ public ContextValue getContextValue() {
+ return this.contextValue;
+ }
+
+ @Override
+ public void setContextValue(ContextValue contextValue) {
+ this.contextValue = contextValue;
+ }
+
+ public String getContextId() {
+ return contextId;
+ }
+
+ public void setContextId(String contextId) {
+ this.contextId = contextId;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextValue.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextValue.java
new file mode 100644
index 0000000..0c18ab6
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/entity/PersistenceContextValue.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.entity;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.persistence.annotation.Ignore;
+
+/**
+ * Created by patinousward on 2020/2/12.
+ */
+@Ignore
+public class PersistenceContextValue implements ContextValue {
+
+ private String keywords;
+ /**
+ * 序列化后的value
+ */
+ private Object value;
+
+ private String valueStr;
+
+ public String getValueStr() {
+ return valueStr;
+ }
+
+ public void setValueStr(String valueStr) {
+ this.valueStr = valueStr;
+ }
+
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+
+ @Override
+ public Object getValue() {
+ return this.value;
+ }
+
+ @Override
+ public void setValue(Object value) {
+ this.value = value;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/exception/ThrowingFunction.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/exception/ThrowingFunction.java
new file mode 100644
index 0000000..7fdac51
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/exception/ThrowingFunction.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.exception;
+
+/**
+ * Created by patinousward on 2019/11/13.
+ */
+@FunctionalInterface
+public interface ThrowingFunction<T, R, E extends Exception> {
+ R accept(T t) throws E;
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextIDListenerPersistence.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextIDListenerPersistence.java
new file mode 100644
index 0000000..c42b279
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextIDListenerPersistence.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextIDListenerPersistence {
+
+ void create(ContextID contextID, ContextIDListenerDomain contextIDListenerDomain) throws CSErrorException;
+
+ void remove(ContextIDListenerDomain contextIDListenerDomain) throws CSErrorException;
+
+ void removeAll(ContextID contextID) throws CSErrorException;
+
+ List<ContextIDListenerDomain> getAll(ContextID contextID) throws CSErrorException;
+
+ ContextIDListenerDomain getBy(ContextIDListenerDomain contextIDListenerDomain) throws CSErrorException;
+
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextIDPersistence.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextIDPersistence.java
new file mode 100644
index 0000000..1140d36
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextIDPersistence.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.listener.ContextIDListener;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextIDPersistence {
+
+ ContextID createContextID(ContextID contextID) throws CSErrorException;
+
+ void deleteContextID(String contextId) throws CSErrorException;
+
+ void updateContextID(ContextID contextID) throws CSErrorException;
+
+ ContextID getContextID(String contextId) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextKeyListenerPersistence.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextKeyListenerPersistence.java
new file mode 100644
index 0000000..7febd96
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextKeyListenerPersistence.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.listener.ContextKeyListener;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextKeyListenerPersistence {
+
+ void create(ContextID contextID, ContextKeyListenerDomain contextKeyListenerDomain) throws CSErrorException;
+
+ void remove(ContextID contextID, ContextKeyListenerDomain contextKeyListenerDomain) throws CSErrorException;
+
+ void removeAll(ContextID contextID) throws CSErrorException;
+
+ List<ContextKeyListenerDomain> getAll(ContextID contextID) throws CSErrorException;
+
+ ContextKeyListenerDomain getBy(ContextKeyListenerDomain contextKeyListenerDomain) throws CSErrorException;
+
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextMapPersistence.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextMapPersistence.java
new file mode 100644
index 0000000..1364e3f
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextMapPersistence.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.listener.ContextKeyListener;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextMapPersistence {
+
+ void create(ContextID contextID, ContextKeyValue contextKeyValue) throws CSErrorException;
+
+ void update(ContextID contextID, ContextKeyValue contextKeyValue) throws CSErrorException;
+
+ ContextKeyValue get(ContextID contextID, ContextKey contextKey) throws CSErrorException;
+
+ List<ContextKeyValue> getAll(ContextID contextID, String key) throws CSErrorException;
+
+ List<ContextKeyValue> getAll(ContextID contextID) throws CSErrorException;
+
+ List<ContextKeyValue> getAll(ContextID contextID, ContextScope contextScope) throws CSErrorException;
+
+ List<ContextKeyValue> getAll(ContextID contextID, ContextType contextType) throws CSErrorException;
+
+ void reset(ContextID contextID, ContextKey contextKey) throws CSErrorException;
+
+ void remove(ContextID contextID, ContextKey contextKey) throws CSErrorException;
+
+ void removeAll(ContextID contextID) throws CSErrorException;
+
+ void removeAll(ContextID contextID, ContextType contextType) throws CSErrorException;
+
+ void removeAll(ContextID contextID, ContextScope contextScope) throws CSErrorException;
+
+ void removeByKeyPrefix(ContextID contextID, String keyPrefix);
+
+ void removeByKeyPrefix(ContextID contextID, ContextType contextType, String keyPrefix);
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextMetricsPersistence.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextMetricsPersistence.java
new file mode 100644
index 0000000..ccf1375
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/ContextMetricsPersistence.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface ContextMetricsPersistence {
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/TransactionManager.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/TransactionManager.java
new file mode 100644
index 0000000..19fcafd
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/TransactionManager.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence;
+
+/**
+ * Created by patinousward on 2020/2/11.
+ */
+public interface TransactionManager {
+
+ Object begin();
+
+ void rollback(Object object);
+
+ void commit(Object object);
+
+ void onTransaction();
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextIDListenerPersistenceImpl.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextIDListenerPersistenceImpl.java
new file mode 100644
index 0000000..cdb1c3c
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextIDListenerPersistenceImpl.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.persistence.dao.ContextIDListenerMapper;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextIDListener;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextIDListenerPersistence;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextIDPersistence;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+@Component
+public class ContextIDListenerPersistenceImpl implements ContextIDListenerPersistence {
+
+ @Autowired
+ private ContextIDListenerMapper contextIDListenerMapper;
+
+ @Autowired
+ private ContextIDPersistence contextIDPersistence;
+
+ @Override
+ public void create(ContextID contextID, ContextIDListenerDomain contextIDListenerDomain) throws CSErrorException {
+ PersistenceContextIDListener listener = new PersistenceContextIDListener();
+ listener.setContextId(contextID.getContextId());
+ listener.setSource(contextIDListenerDomain.getSource());
+ contextIDListenerMapper.createIDListener(listener);
+ }
+
+ @Override
+ public void remove(ContextIDListenerDomain contextIDListenerDomain) throws CSErrorException {
+ // TODO: 2020/2/17
+ PersistenceContextIDListener listener = new PersistenceContextIDListener();
+ listener.setContextId(contextIDListenerDomain.getContextID().getContextId());
+ listener.setSource(contextIDListenerDomain.getSource());
+ contextIDListenerMapper.remove(listener);
+ }
+
+ @Override
+ public void removeAll(ContextID contextID) throws CSErrorException {
+ contextIDListenerMapper.removeAll(contextID);
+ }
+
+ @Override
+ public List<ContextIDListenerDomain> getAll(ContextID contextID) throws CSErrorException {
+ // 根据id返回一堆的domain
+ ContextID complete = contextIDPersistence.getContextID(contextID.getContextId());
+ List<PersistenceContextIDListener> listeners = contextIDListenerMapper.getAll(contextID);
+ List<ContextIDListenerDomain> domains = listeners.stream().map(l -> pDomainToCommon(l, complete)).collect(Collectors.toList());
+ return domains;
+ }
+
+ public ContextIDListenerDomain pDomainToCommon(PersistenceContextIDListener listener, ContextID contextID) {
+ CommonContextIDListenerDomain domain = new CommonContextIDListenerDomain();
+ domain.setContextID(contextID);
+ domain.setSource(listener.getSource());
+ return domain;
+ }
+
+ @Override
+ public ContextIDListenerDomain getBy(ContextIDListenerDomain contextIDListenerDomain) throws CSErrorException {
+ //根据id 和source 返回响应的ContextIDListenerDomain
+ return contextIDListenerDomain;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextIDPersistenceImpl.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextIDPersistenceImpl.java
new file mode 100644
index 0000000..c7b26c3
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextIDPersistenceImpl.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.listener.ContextIDListener;
+import com.webank.wedatasphere.linkis.cs.persistence.dao.ContextIDMapper;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.ExtraFieldClass;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextID;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextIDPersistence;
+import com.webank.wedatasphere.linkis.cs.persistence.util.PersistenceUtils;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.apache.commons.math3.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+
+/**
+ * Created by patinousward on 2020/2/12.
+ */
+@Component
+public class ContextIDPersistenceImpl implements ContextIDPersistence {
+
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ @Autowired
+ private ContextIDMapper contextIDMapper;
+
+ private Class<PersistenceContextID> pClass = PersistenceContextID.class;
+
+ private ObjectMapper json = BDPJettyServerHelper.jacksonJson();
+
+ @Override
+ public ContextID createContextID(ContextID contextID) throws CSErrorException {
+ try {
+ Pair<PersistenceContextID, ExtraFieldClass> pContextID = PersistenceUtils.transfer(contextID, pClass);
+ pContextID.getFirst().setSource(json.writeValueAsString(pContextID.getSecond()));
+ contextIDMapper.createContextID(pContextID.getFirst());
+ contextID.setContextId(pContextID.getFirst().getContextId());
+ return contextID;
+ } catch (JsonProcessingException e) {
+ logger.error("writeAsJson failed:", e);
+ throw new CSErrorException(97000, e.getMessage());
+ }
+ }
+
+ @Override
+ public void deleteContextID(String contextId) {
+ contextIDMapper.deleteContextID(contextId);
+ }
+
+ @Override
+ public void updateContextID(ContextID contextID) throws CSErrorException {
+ //contextId和source没有设置更新点
+ Pair<PersistenceContextID, ExtraFieldClass> pContextID = PersistenceUtils.transfer(contextID, pClass);
+ contextIDMapper.updateContextID(pContextID.getFirst());
+ }
+
+ @Override
+ public ContextID getContextID(String contextId) throws CSErrorException {
+ try {
+ PersistenceContextID pContextID = contextIDMapper.getContextID(contextId);
+ if(pContextID == null) return null;
+ ExtraFieldClass extraFieldClass = json.readValue(pContextID.getSource(), ExtraFieldClass.class);
+ ContextID contextID = PersistenceUtils.transfer(extraFieldClass, pContextID);
+ return contextID;
+ } catch (IOException e) {
+ logger.error("readJson failed:", e);
+ throw new CSErrorException(97000, e.getMessage());
+ }
+ }
+
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextKeyListenerPersistenceImpl.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextKeyListenerPersistenceImpl.java
new file mode 100644
index 0000000..184bd43
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextKeyListenerPersistenceImpl.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.persistence.dao.ContextKeyListenerMapper;
+import com.webank.wedatasphere.linkis.cs.persistence.dao.ContextMapMapper;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyListener;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextKeyListenerPersistence;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextMapPersistence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+@Component
+public class ContextKeyListenerPersistenceImpl implements ContextKeyListenerPersistence {
+
+ @Autowired
+ private ContextKeyListenerMapper contextKeyListenerMapper;
+
+ @Autowired
+ private ContextMapMapper contextMapMapper;
+
+ @Autowired
+ private ContextMapPersistence contextMapPersistence;
+
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ @Override
+ public void create(ContextID contextID, ContextKeyListenerDomain contextKeyListenerDomain) throws CSErrorException {
+ PersistenceContextKeyListener listener = new PersistenceContextKeyListener();
+ listener.setSource(contextKeyListenerDomain.getSource());
+ // contextKey 去数据库中查出响应的id
+ PersistenceContextKeyValue contextMap = contextMapMapper.getContextMap(contextID, contextKeyListenerDomain.getContextKey());
+ listener.setKeyId(contextMap.getId());
+ contextKeyListenerMapper.createKeyListener(listener);
+ }
+
+ @Override
+ public void remove(ContextID contextID, ContextKeyListenerDomain contextKeyListenerDomain) throws CSErrorException {
+ PersistenceContextKeyValue contextMap = contextMapMapper.getContextMap(contextID, contextKeyListenerDomain.getContextKey());
+ Integer keyId = contextMap.getId();
+ contextKeyListenerMapper.remove(contextKeyListenerDomain, keyId);
+ }
+
+ @Override
+ public void removeAll(ContextID contextID) throws CSErrorException {
+ //找到contextID 对应的所有map id
+ List<Integer> keyIds = contextMapMapper.getAllContextMapByContextID(contextID).stream()
+ .map(PersistenceContextKeyValue::getId).collect(Collectors.toList());
+ contextKeyListenerMapper.removeAll(keyIds);
+ }
+
+ @Override
+ public List<ContextKeyListenerDomain> getAll(ContextID contextID) throws CSErrorException {
+ List<PersistenceContextKeyValue> pKVs = contextMapMapper.getAllContextMapByContextID(contextID);
+ List<Integer> keyIds = contextMapMapper.getAllContextMapByContextID(contextID).stream()
+ .map(PersistenceContextKeyValue::getId).collect(Collectors.toList());
+ ArrayList<ContextKeyListenerDomain> domains = new ArrayList<>();
+ if (!keyIds.isEmpty()) {
+ logger.info("fetch %s keyIds by contextId %s", keyIds.size(), contextID.getContextId());
+ List<PersistenceContextKeyListener> listeners = contextKeyListenerMapper.getAll(keyIds);
+ for (PersistenceContextKeyListener listener : listeners) {
+ domains.add(pDomainToCommon(listener, contextID, pKVs));
+ }
+ }
+ return domains;
+ }
+
+ public ContextKeyListenerDomain pDomainToCommon(PersistenceContextKeyListener listener, ContextID contextID, List<PersistenceContextKeyValue> pKVs) throws CSErrorException {
+ CommonContextKeyListenerDomain domain = new CommonContextKeyListenerDomain();
+ domain.setSource(listener.getSource());
+ Optional<PersistenceContextKeyValue> first = pKVs.stream().filter(kv -> kv.getId().equals(listener.getId())).findFirst();
+ if (first.isPresent()) {
+ ContextKeyValue contextKeyValue = contextMapPersistence.get(contextID, first.get().getContextKey());
+ domain.setContextKey(contextKeyValue.getContextKey());
+ }
+ return domain;
+ }
+
+ @Override
+ public ContextKeyListenerDomain getBy(ContextKeyListenerDomain contextKeyListenerDomain) throws CSErrorException {
+
+ return contextKeyListenerDomain;
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextMapPersistenceImpl.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextMapPersistenceImpl.java
new file mode 100644
index 0000000..1ee3966
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextMapPersistenceImpl.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper;
+import com.webank.wedatasphere.linkis.cs.persistence.dao.ContextMapMapper;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.ExtraFieldClass;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKey;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextValue;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextMapPersistence;
+import com.webank.wedatasphere.linkis.cs.persistence.util.PersistenceUtils;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.apache.commons.math3.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Created by patinousward on 2020/2/13.
+ */
+@Component
+public class ContextMapPersistenceImpl implements ContextMapPersistence {
+
+ @Autowired
+ private ContextMapMapper contextMapMapper;
+
+ private Class<PersistenceContextKey> pKClass = PersistenceContextKey.class;
+
+ private Class<PersistenceContextValue> pVClass = PersistenceContextValue.class;
+
+ private Class<PersistenceContextKeyValue> pKVClass = PersistenceContextKeyValue.class;
+
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private ObjectMapper json = BDPJettyServerHelper.jacksonJson();
+
+ private final SerializationHelper serialHelper = ContextSerializationHelper.getInstance();
+
+ @Override
+ public void create(ContextID contextID, ContextKeyValue kV) throws CSErrorException {
+ // TODO: 2020/2/17 keywords 如何合并
+ try {
+ Pair<PersistenceContextKey, ExtraFieldClass> pK = PersistenceUtils.transfer(kV.getContextKey(), pKClass);
+ Pair<PersistenceContextValue, ExtraFieldClass> pV = PersistenceUtils.transfer(kV.getContextValue(), pVClass);
+ Pair<PersistenceContextKeyValue, ExtraFieldClass> pKV = PersistenceUtils.transfer(kV, pKVClass);
+ pV.getFirst().setValueStr(serialHelper.serialize(kV.getContextValue().getValue()));
+ pKV.getSecond().addSub(pK.getSecond());
+ pKV.getSecond().addSub(pV.getSecond());
+ pKV.getFirst().setProps(json.writeValueAsString(pKV.getSecond()));
+ pKV.getFirst().setContextId(contextID.getContextId());
+ pKV.getFirst().setContextKey(pK.getFirst());
+ pKV.getFirst().setContextValue(pV.getFirst());
+ contextMapMapper.createMap(pKV.getFirst());
+ } catch (JsonProcessingException e) {
+ logger.error("writeAsJson failed:", e);
+ throw new CSErrorException(97000, e.getMessage());
+ }
+ }
+
+ @Override
+ public void update(ContextID contextID, ContextKeyValue kV) throws CSErrorException {
+ //根据contextId和key 进行更新
+ Pair<PersistenceContextKey, ExtraFieldClass> pK = PersistenceUtils.transfer(kV.getContextKey(), pKClass);
+ Pair<PersistenceContextValue, ExtraFieldClass> pV = PersistenceUtils.transfer(kV.getContextValue(), pVClass);
+ Pair<PersistenceContextKeyValue, ExtraFieldClass> pKV = PersistenceUtils.transfer(kV, pKVClass);
+ Object value = kV.getContextValue().getValue();
+ if (value != null) {
+ pV.getFirst().setValueStr(serialHelper.serialize(value));
+ }
+ pKV.getFirst().setContextKey(pK.getFirst());
+ pKV.getFirst().setContextValue(pV.getFirst());
+ pKV.getFirst().setContextId(contextID.getContextId());
+ contextMapMapper.updateMap(pKV.getFirst());
+ }
+
+ @Override
+ public ContextKeyValue get(ContextID contextID, ContextKey contextKey) throws CSErrorException {
+ //根据contextId 和key,获取到一个ContextKeyValue
+ PersistenceContextKeyValue pKV = contextMapMapper.getContextMap(contextID, contextKey);
+ if (pKV == null) return null;
+ return transfer(pKV);
+ }
+
+ private ContextKeyValue transfer(PersistenceContextKeyValue pKV) throws CSErrorException {
+ try {
+ // TODO: 2020/2/14 null return
+ PersistenceContextKey pK = (PersistenceContextKey) pKV.getContextKey();
+ PersistenceContextValue pV = (PersistenceContextValue) pKV.getContextValue();
+ ExtraFieldClass extraFieldClass = json.readValue(pKV.getProps(), ExtraFieldClass.class);
+ ContextKey key = PersistenceUtils.transfer(extraFieldClass.getOneSub(0), pK);
+ ContextValue value = PersistenceUtils.transfer(extraFieldClass.getOneSub(1), pV);
+ if(value != null){
+ value.setValue(serialHelper.deserialize(pV.getValueStr()));
+ }
+ ContextKeyValue kv = PersistenceUtils.transfer(extraFieldClass, pKV);
+ kv.setContextKey(key);
+ kv.setContextValue(value);
+ return kv;
+ } catch (IOException e) {
+ logger.error("readJson failed:", e);
+ throw new CSErrorException(97000, e.getMessage());
+ }
+ }
+
+ @Override
+ public List<ContextKeyValue> getAll(ContextID contextID, String key) {
+ //模糊匹配key
+ List<PersistenceContextKeyValue> pKVs = contextMapMapper.getAllContextMapByKey(contextID, key);
+ return pKVs.stream().map(PersistenceUtils.map(this::transfer)).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<ContextKeyValue> getAll(ContextID contextID) {
+ List<PersistenceContextKeyValue> pKVs = contextMapMapper.getAllContextMapByContextID(contextID);
+ return pKVs.stream().map(PersistenceUtils.map(this::transfer)).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<ContextKeyValue> getAll(ContextID contextID, ContextScope contextScope) {
+ List<PersistenceContextKeyValue> pKVs = contextMapMapper.getAllContextMapByScope(contextID, contextScope);
+ return pKVs.stream().map(PersistenceUtils.map(this::transfer)).collect(Collectors.toList());
+ }
+
+ @Override
+ public List<ContextKeyValue> getAll(ContextID contextID, ContextType contextType) {
+ List<PersistenceContextKeyValue> pKVs = contextMapMapper.getAllContextMapByType(contextID, contextType);
+ return pKVs.stream().map(PersistenceUtils.map(this::transfer)).collect(Collectors.toList());
+ }
+
+ @Override
+ public void reset(ContextID contextID, ContextKey contextKey) {
+
+ }
+
+ @Override
+ public void remove(ContextID contextID, ContextKey contextKey) {
+ contextMapMapper.removeContextMap(contextID, contextKey);
+ }
+
+ @Override
+ public void removeAll(ContextID contextID) {
+ contextMapMapper.removeAllContextMapByContextID(contextID);
+ }
+
+ @Override
+ public void removeAll(ContextID contextID, ContextType contextType) {
+ contextMapMapper.removeAllContextMapByType(contextID, contextType);
+ }
+
+ @Override
+ public void removeAll(ContextID contextID, ContextScope contextScope) {
+ contextMapMapper.removeAllContextMapByScope(contextID, contextScope);
+ }
+
+ @Override
+ public void removeByKeyPrefix(ContextID contextID, String keyPrefix) {
+ contextMapMapper.removeByKeyPrefix(contextID,keyPrefix);
+ }
+
+ @Override
+ public void removeByKeyPrefix(ContextID contextID, ContextType contextType, String keyPrefix) {
+ contextMapMapper.removeByKeyPrefixAndContextType(contextID,contextType,keyPrefix);
+ }
+
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextMetricsPersistenceImpl.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextMetricsPersistenceImpl.java
new file mode 100644
index 0000000..d4946bf
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/ContextMetricsPersistenceImpl.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence.impl;
+
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextMetricsPersistence;
+import org.springframework.stereotype.Component;
+
+/**
+ * Created by patinousward on 2020/2/23.
+ */
+@Component
+public class ContextMetricsPersistenceImpl implements ContextMetricsPersistence {
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/TransactionManagerImpl.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/TransactionManagerImpl.java
new file mode 100644
index 0000000..aee60de
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/persistence/impl/TransactionManagerImpl.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.persistence.impl;
+
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.TransactionManager;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+
+/**
+ * Created by patinousward on 2020/2/17.
+ */
+@Component
+public class TransactionManagerImpl implements TransactionManager {
+
+ @Autowired
+ private PlatformTransactionManager txManager;
+
+ @Override
+ public Object begin() {
+ return txManager.getTransaction(new DefaultTransactionDefinition());
+ }
+
+ @Override
+ public void rollback(Object object) {
+ TransactionStatus status = (TransactionStatus) object;
+ txManager.rollback(status);
+ }
+
+ @Override
+ public void commit(Object object) {
+ TransactionStatus status = (TransactionStatus) object;
+ txManager.commit(status);
+ }
+
+ @Override
+ public void onTransaction() {
+ // TODO: 2020/2/17
+ }
+}
diff --git a/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/util/PersistenceUtils.java b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/util/PersistenceUtils.java
new file mode 100644
index 0000000..8e29dc2
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/java/com/webank/wedatasphere/linkis/cs/persistence/util/PersistenceUtils.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.persistence.util;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSWarnException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper;
+import com.webank.wedatasphere.linkis.cs.persistence.annotation.Ignore;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.ExtraFieldClass;
+import com.webank.wedatasphere.linkis.cs.persistence.exception.ThrowingFunction;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.apache.commons.math3.util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanUtils;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Created by patinousward on 2020/2/13.
+ */
+public class PersistenceUtils {
+
+ private static ObjectMapper json = BDPJettyServerHelper.jacksonJson();
+
+ private static final Logger logger = LoggerFactory.getLogger(PersistenceUtils.class);
+
+ private static String generateGetMethod(Field field) {
+ String fieldName = field.getName();
+ return String.format("get%s%s", fieldName.substring(0, 1).toUpperCase(), fieldName.substring(1));
+ }
+
+ public static String generateSetMethod(String fieldName) {
+ return String.format("set%s%s", fieldName.substring(0, 1).toUpperCase(), fieldName.substring(1));
+ }
+
+ private static boolean canIgnore(Field field) {
+ return field.getAnnotation(Ignore.class) != null;
+ }
+
+ private static List<String> getIgnoreFieldName(Class<?> clazz) {
+ if (clazz.getAnnotation(Ignore.class) != null) {
+ return Arrays.stream(clazz.getDeclaredFields()).map(Field::getName).collect(Collectors.toList());
+ } else {
+ return Arrays.stream(clazz.getDeclaredFields())
+ .filter(PersistenceUtils::canIgnore).map(Field::getName).collect(Collectors.toList());
+ }
+ }
+
+ public static <T, S> Pair<S, ExtraFieldClass> transfer(T t, Class<S> sClass) throws CSErrorException {
+ try {
+ ExtraFieldClass extraFieldClass = new ExtraFieldClass();
+ S s = sClass.newInstance();
+ BeanUtils.copyProperties(t, s);
+ Class<?> tClass = t.getClass();
+ extraFieldClass.setClassName(tClass.getName());
+ List<String> canIgnore = getIgnoreFieldName(sClass);
+ for (Field field : tClass.getDeclaredFields()) {
+ if (!canIgnore.contains(field.getName())) {
+ Method method = tClass.getMethod(generateGetMethod(field));
+ if (null != method.invoke(t)) {
+ //field.getType().getName() 无法拿到子类的类型
+ Object invoke = method.invoke(t);
+ extraFieldClass.addFieldName(field.getName());
+ if (invoke == null) {
+ extraFieldClass.addFieldType(field.getType().getName());
+ } else {
+ extraFieldClass.addFieldType(invoke.getClass().getName());
+ }
+ extraFieldClass.addFieldValue(invoke);
+ }
+ }
+ }
+ return new Pair<>(s, extraFieldClass);
+ } catch (Exception e) {
+ throw new CSErrorException(97000, "transfer bean failed:", e);
+ }
+ }
+
+ public static <T, S> T transfer(ExtraFieldClass extraFieldClass, S s) throws CSErrorException {
+ if (s == null) return null;
+ try {
+ Class<?> tClass = Class.forName(extraFieldClass.getClassName());
+ T t = (T) tClass.newInstance();
+ BeanUtils.copyProperties(s, t);
+ for (int i = 0; i < extraFieldClass.getFieldNames().size(); i++) {
+ Field field = tClass.getDeclaredField(extraFieldClass.getOneFieldName(i));
+ field.setAccessible(true);
+ if (LONG_TYP.equals(extraFieldClass.getOneFieldType(i))) {
+ Long value = new Long(extraFieldClass.getOneFieldValue(i).toString());
+ field.set(t, value);
+ } else if (Enum.class.isAssignableFrom(Class.forName(extraFieldClass.getOneFieldType(i)))) {
+ //反序列化支持枚举类
+ Class<?> enumClass = Class.forName(extraFieldClass.getOneFieldType(i));
+ Method valueOf = enumClass.getMethod("valueOf", String.class);
+ Object invoke = valueOf.invoke(null, extraFieldClass.getOneFieldValue(i));
+ field.set(t, invoke);
+ } else if (!BeanUtils.isSimpleProperty(Class.forName(extraFieldClass.getOneFieldType(i)))) {
+ //非基本类型的话,使用jackson进行反序列化 // TODO: 2020/3/5 这里属性的序列化and反序列化最好修改为utils的序列化器
+ Object o = json.convertValue(extraFieldClass.getOneFieldValue(i), Class.forName(extraFieldClass.getOneFieldType(i)));
+ field.set(t, o);
+ } else {
+ field.set(t, extraFieldClass.getOneFieldValue(i));
+ }
+ }
+ return t;
+ } catch (Exception e) {
+ throw new CSErrorException(97000, "transfer bean failed:", e);
+ }
+ }
+
+ public static <T, R, E extends Exception> Function<T, R> map(
+ ThrowingFunction<T, R, E> throwingFunction) {
+ return i -> {
+ try {
+ return throwingFunction.accept(i);
+ } catch (Exception e) {
+ throw new CSWarnException(97000, "execute failed,reason:", e);
+ }
+ };
+ }
+
+ private static final String LONG_TYP = "java.lang.Long";
+
+ // TODO: 2020/5/15 去掉重复的
+ public static final SerializationHelper SERIALIZE = ContextSerializationHelper.getInstance();
+
+ public static String serialize(Object o) throws CSErrorException {
+ if (o instanceof String) {
+ return (String) o;
+ }
+ return SERIALIZE.serialize(o);
+ }
+
+ public static <T> T deserialize(String Str) throws CSErrorException {
+ return (T) SERIALIZE.deserialize(Str);
+ }
+
+}
diff --git a/contextservice/cs-persistence/src/main/resources/cs_ddl.sql b/contextservice/cs-persistence/src/main/resources/cs_ddl.sql
new file mode 100644
index 0000000..73f6cbb
--- /dev/null
+++ b/contextservice/cs-persistence/src/main/resources/cs_ddl.sql
@@ -0,0 +1,90 @@
+--
+-- Copyright 2019 WeBank
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SET FOREIGN_KEY_CHECKS=0;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_history
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_history`;
+CREATE TABLE `linkis_cs_context_history` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `context_id` int(11) DEFAULT NULL,
+ `source` text,
+ `context_type` varchar(32) DEFAULT NULL,
+ `history_json` text,
+ `keyword` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `keyword` (`keyword`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_id
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_id`;
+CREATE TABLE `linkis_cs_context_id` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `user` varchar(32) DEFAULT NULL,
+ `application` varchar(32) DEFAULT NULL,
+ `source` varchar(255) DEFAULT NULL,
+ `expire_type` varchar(32) DEFAULT NULL,
+ `expire_time` datetime DEFAULT NULL,
+ `instance` varchar(32) DEFAULT NULL,
+ `backup_instance` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `instance` (`instance`),
+ KEY `backup_instance` (`backup_instance`),
+ KEY `instance_2` (`instance`,`backup_instance`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_listener
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_listener`;
+CREATE TABLE `linkis_cs_context_listener` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `listener_source` varchar(255) DEFAULT NULL,
+ `context_id` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_map
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_map`;
+CREATE TABLE `linkis_cs_context_map` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `key` varchar(255) DEFAULT NULL,
+ `context_scope` varchar(32) DEFAULT NULL,
+ `context_type` varchar(32) DEFAULT NULL,
+ `props` text,
+ `value` text,
+ `context_id` int(11) DEFAULT NULL,
+ `keywords` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `keywords` (`keywords`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_map_listener
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_map_listener`;
+CREATE TABLE `linkis_cs_context_map_listener` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `listener_source` varchar(255) DEFAULT NULL,
+ `key_id` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
\ No newline at end of file
diff --git a/contextservice/cs-search/pom.xml b/contextservice/cs-search/pom.xml
new file mode 100644
index 0000000..6bbf43f
--- /dev/null
+++ b/contextservice/cs-search/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-search</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-cache</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cloudRPC</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>RELEASE</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>2.0.2-beta</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/ContextSearch.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/ContextSearch.java
new file mode 100644
index 0000000..02e2324
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/ContextSearch.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+
+import java.util.List;
+import java.util.Map;
+
+public interface ContextSearch {
+
+ List<ContextKeyValue> search(ContextCacheService contextCacheService, ContextID contextID, Map<Object, Object> conditionMap) throws ContextSearchFailedException;
+ List<ContextKeyValue> search(ContextCacheService contextCacheService, ContextID contextID, Condition condition) throws ContextSearchFailedException;
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/DefaultContextSearch.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/DefaultContextSearch.java
new file mode 100644
index 0000000..c43f6f6
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/DefaultContextSearch.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.construction.ConditionParser;
+import com.webank.wedatasphere.linkis.cs.condition.impl.*;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+import com.webank.wedatasphere.linkis.cs.execution.ConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.impl.*;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class DefaultContextSearch implements ContextSearch {
+
+ private static Logger logger = LoggerFactory.getLogger(DefaultContextSearch.class);
+
+ @Override
+ public List<ContextKeyValue> search(ContextCacheService contextCacheService, ContextID contextID, Map<Object, Object> conditionMap) throws ContextSearchFailedException {
+ logger.info("Got search condition: \n" + BDPJettyServerHelper.gson().toJson(conditionMap));
+ ConditionParser conditionParser = ConditionParser.parserMap.get(conditionMap.get("type"));
+ return search(contextCacheService, contextID, conditionParser.parse(conditionMap));
+ }
+
+ @Override
+ public List<ContextKeyValue> search(ContextCacheService contextCacheService, ContextID contextID, Condition condition) throws ContextSearchFailedException {
+ return getExecution(contextCacheService, contextID, condition).execute();
+ }
+
+ private ConditionExecution getExecution(ContextCacheService contextCacheService, ContextID contextID, Condition condition) throws ContextSearchFailedException {
+ if(condition instanceof ContextTypeCondition){
+ return new ContextTypeConditionExecution((ContextTypeCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof ContextScopeCondition){
+ return new ContextScopeConditionExecution((ContextScopeCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof RegexCondition){
+ return new RegexConditionExecution((RegexCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof ContainsCondition){
+ return new ContainsConditionExecution((ContainsCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof AndCondition){
+ return new AndConditionExecution((AndCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof OrCondition){
+ return new OrConditionExecution((OrCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof NotCondition){
+ return new NotConditionExecution((NotCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof NearestCondition){
+ return new NearestConditionExecution((NearestCondition) condition, contextCacheService, contextID);
+ } else if(condition instanceof ContextValueTypeCondition){
+ return new ContextValueTypeConditionExecution((ContextValueTypeCondition) condition, contextCacheService, contextID);
+ }
+ throw new ContextSearchFailedException(1200001, "Unknown Condition Type");
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/AbstractCommonCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/AbstractCommonCondition.java
new file mode 100644
index 0000000..af6277c
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/AbstractCommonCondition.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition;
+
+import com.webank.wedatasphere.linkis.cs.condition.impl.AndCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NotCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.OrCondition;
+
+public abstract class AbstractCommonCondition implements Condition{
+
+ @Override
+ public Condition and(Condition right) {
+ return new AndCondition(this, right);
+ }
+
+ @Override
+ public Condition or(Condition right) {
+ return new OrCondition(this, right);
+ }
+
+ @Override
+ public Condition not() {
+ return new NotCondition(this);
+ }
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/AtomicCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/AtomicCondition.java
new file mode 100644
index 0000000..c0e1bb7
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/AtomicCondition.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition;
+
+public abstract class AtomicCondition extends AbstractCommonCondition {
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/BinaryLogicCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/BinaryLogicCondition.java
new file mode 100644
index 0000000..eae2d21
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/BinaryLogicCondition.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition;
+
+public abstract class BinaryLogicCondition extends AbstractCommonCondition{
+
+ Condition left;
+ Condition right;
+
+ public BinaryLogicCondition(Condition left, Condition right) {
+ this.left = left;
+ this.right = right;
+ }
+
+ public void shift(){
+ Condition tmp = this.left;
+ this.left = this.right;
+ this.right = tmp;
+ }
+
+ public Condition getLeft() {
+ return left;
+ }
+
+ public void setLeft(Condition left) {
+ this.left = left;
+ }
+
+ public Condition getRight() {
+ return right;
+ }
+
+ public void setRight(Condition right) {
+ right = right;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/Condition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/Condition.java
new file mode 100644
index 0000000..a252644
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/Condition.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition;
+
+public interface Condition {
+
+ Condition and(Condition right);
+ Condition or(Condition right);
+ Condition not();
+ ConditionType getConditionType();
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/ConditionType.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/ConditionType.java
new file mode 100644
index 0000000..12ac221
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/ConditionType.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition;
+
+public enum ConditionType {
+ Regex,
+ Contains,
+ Equals,
+ Logic
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/UnaryLogicCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/UnaryLogicCondition.java
new file mode 100644
index 0000000..2d33912
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/UnaryLogicCondition.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition;
+
+public abstract class UnaryLogicCondition extends AbstractCommonCondition{
+ Condition origin;
+
+ public UnaryLogicCondition(Condition origin) {
+ this.origin = origin;
+ }
+
+ public Condition getOrigin() {
+ return origin;
+ }
+
+ public void setOrigin(Condition origin) {
+ this.origin = origin;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/AndConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/AndConditionParser.java
new file mode 100644
index 0000000..8c48525
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/AndConditionParser.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.AndCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+
+import java.util.Map;
+
+public class AndConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ Map<Object, Object> left = (Map<Object, Object>) conditionMap.get("left");
+ Map<Object, Object> right = (Map<Object, Object>) conditionMap.get("right");
+ return new AndCondition(
+ parserMap.get(left.get("type")).parse(left),
+ parserMap.get(right.get("type")).parse(right)
+ );
+ }
+
+ @Override
+ public String getName() {
+ return "And";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionBuilder.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionBuilder.java
new file mode 100644
index 0000000..f0fb1a2
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionBuilder.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+
+import java.util.Collection;
+
+public interface ConditionBuilder {
+
+ public static ConditionBuilder newBuilder(){
+ return new ConditionBuilderImpl();
+ }
+
+ ConditionBuilder contextTypes(Collection<ContextType> contextTypes);
+ ConditionBuilder contextScopes(Collection<ContextScope> contextScopes);
+ ConditionBuilder regex(String regex);
+ ConditionBuilder contains(String value);
+ Condition build();
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionBuilderImpl.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionBuilderImpl.java
new file mode 100644
index 0000000..daff5c7
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionBuilderImpl.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContainsCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextScopeCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.RegexCondition;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Collection;
+
+public class ConditionBuilderImpl implements ConditionBuilder{
+
+ Collection<ContextType> contextTypes;
+ Collection<ContextScope> contextScopes;
+ String regex;
+ String containsValue;
+
+ @Override
+ public ConditionBuilder contextTypes(Collection<ContextType> contextTypes) {
+ this.contextTypes = contextTypes;
+ return this;
+ }
+
+ @Override
+ public ConditionBuilder contextScopes(Collection<ContextScope> contextScopes) {
+ this.contextScopes = contextScopes;
+ return this;
+ }
+
+ @Override
+ public ConditionBuilder regex(String regex) {
+ this.regex = regex;
+ return this;
+ }
+
+ @Override
+ public ConditionBuilder contains(String value) {
+ this.containsValue = value;
+ return this;
+ }
+
+ @Override
+ public Condition build() {
+ Condition condition = null;
+ if(CollectionUtils.isNotEmpty(contextTypes)){
+ for(ContextType contextType : contextTypes){
+ Condition subCondition = new ContextTypeCondition(contextType);
+ condition = combineCondition(condition, subCondition, true);
+ }
+ }
+ Condition conditionContextScope = null;
+ if(CollectionUtils.isNotEmpty(contextScopes)){
+ for(ContextScope contextScope : contextScopes){
+ Condition subCondition = new ContextScopeCondition(contextScope);
+ conditionContextScope = combineCondition(conditionContextScope, subCondition, true);
+ }
+ condition = combineCondition(condition, conditionContextScope, false);
+ }
+
+ if(StringUtils.isNotBlank(regex)){
+ Condition subCondition = new RegexCondition(regex);
+ condition = combineCondition(condition, subCondition, false);
+ }
+ if(StringUtils.isNotBlank(containsValue)){
+ Condition subCondition = new ContainsCondition(containsValue);
+ condition = combineCondition(condition, subCondition, false);
+ }
+ return condition;
+ }
+
+ private Condition combineCondition(Condition condition, Condition subCondition, boolean or) {
+ if(condition == null){
+ condition = subCondition;
+ } else if (or){
+ condition = condition.or(subCondition);
+ } else {
+ condition = condition.and(subCondition);
+ }
+ return condition;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionParser.java
new file mode 100644
index 0000000..e277b84
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ConditionParser.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public interface ConditionParser {
+
+ public static Map<String, ConditionParser> parserMap = new HashMap<String, ConditionParser>(){{
+ List<ConditionParser> conditionParsers = Lists.newArrayList(
+ new RegexConditionParser(),
+ new ContainsConditionParser(),
+ new ContextTypeConditionParser(),
+ new ContextScopeConditionParser(),
+ new AndConditionParser(),
+ new OrConditionParser(),
+ new NotConditionParser(),
+ new NearestConditionParser(),
+ new ContextValueTypeConditionParser()
+ );
+ for(ConditionParser conditionParser : conditionParsers){
+ put(conditionParser.getName(), conditionParser);
+ }
+ }};
+
+
+ Condition parse(Map<Object, Object> conditionMap);
+ String getName();
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContainsConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContainsConditionParser.java
new file mode 100644
index 0000000..32374df
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContainsConditionParser.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContainsCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.RegexCondition;
+
+import java.util.Map;
+
+public class ContainsConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ return new ContainsCondition((String) conditionMap.get("value"));
+ }
+
+ @Override
+ public String getName() {
+ return "Contains";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextScopeConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextScopeConditionParser.java
new file mode 100644
index 0000000..4a9ebe8
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextScopeConditionParser.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextScopeCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+
+import java.util.Map;
+
+public class ContextScopeConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ return new ContextScopeCondition(ContextScope.valueOf((String) conditionMap.get("contextScope")));
+ }
+
+ @Override
+ public String getName() {
+ return "ContextScope";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextTypeConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextTypeConditionParser.java
new file mode 100644
index 0000000..41acf2f
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextTypeConditionParser.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContainsCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+
+import java.util.Map;
+
+public class ContextTypeConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ return new ContextTypeCondition(ContextType.valueOf((String) conditionMap.get("contextType")));
+ }
+
+ @Override
+ public String getName() {
+ return "ContextType";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextValueTypeConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextValueTypeConditionParser.java
new file mode 100644
index 0000000..6043316
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/ContextValueTypeConditionParser.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextValueTypeCondition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public class ContextValueTypeConditionParser implements ConditionParser{
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextValueTypeConditionParser.class);
+
+
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+
+ Class contextValueType = Object.class;
+ try {
+ contextValueType = Class.forName((String) conditionMap.get("contextValueType"));
+ } catch (ClassNotFoundException e) {
+ logger.error("Cannot find contextValueType:" + conditionMap.get("contextValueType"));
+ }
+ return new ContextValueTypeCondition(contextValueType);
+ }
+
+ @Override
+ public String getName() {
+ return "ContextValueType";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/NearestConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/NearestConditionParser.java
new file mode 100644
index 0000000..d13fd1d
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/NearestConditionParser.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NearestCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NotCondition;
+
+import java.util.Map;
+
+public class NearestConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ Map<Object, Object> origin = (Map<Object, Object>) conditionMap.get("origin");
+ return new NearestCondition(
+ parserMap.get(origin.get("type")).parse(origin),
+ conditionMap.get("currentNode").toString(),
+ Integer.parseInt(conditionMap.get("number").toString()),
+ Boolean.parseBoolean(conditionMap.get("upstreamOnly").toString())
+ );
+ }
+
+ @Override
+ public String getName() {
+ return "Nearest";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/NotConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/NotConditionParser.java
new file mode 100644
index 0000000..4626cb6
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/NotConditionParser.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NotCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.OrCondition;
+
+import java.util.Map;
+
+public class NotConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ Map<Object, Object> origin = (Map<Object, Object>) conditionMap.get("origin");
+ return new NotCondition(
+ parserMap.get(origin.get("type")).parse(origin)
+ );
+ }
+
+ @Override
+ public String getName() {
+ return "Not";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/OrConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/OrConditionParser.java
new file mode 100644
index 0000000..ff916fb
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/OrConditionParser.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.AndCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.OrCondition;
+
+import java.util.Map;
+
+public class OrConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ Map<Object, Object> left = (Map<Object, Object>) conditionMap.get("left");
+ Map<Object, Object> right = (Map<Object, Object>) conditionMap.get("right");
+ return new OrCondition(
+ parserMap.get(left.get("type")).parse(left),
+ parserMap.get(right.get("type")).parse(right)
+ );
+ }
+
+ @Override
+ public String getName() {
+ return "Or";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/RegexConditionParser.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/RegexConditionParser.java
new file mode 100644
index 0000000..0258e8c
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/construction/RegexConditionParser.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.construction;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.RegexCondition;
+
+import java.util.Map;
+
+public class RegexConditionParser implements ConditionParser{
+ @Override
+ public Condition parse(Map<Object, Object> conditionMap) {
+ return new RegexCondition((String) conditionMap.get("regex"));
+ }
+
+ @Override
+ public String getName() {
+ return "Regex";
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/AndCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/AndCondition.java
new file mode 100644
index 0000000..7455d4d
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/AndCondition.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.condition.BinaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+
+public class AndCondition extends BinaryLogicCondition {
+
+ public AndCondition(Condition left, Condition right) {
+ super(left, right);
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Logic;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContainsCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContainsCondition.java
new file mode 100644
index 0000000..5dca527
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContainsCondition.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.condition.AtomicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+
+public class ContainsCondition extends AtomicCondition {
+ String value;
+
+ public ContainsCondition(String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Contains;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextScopeCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextScopeCondition.java
new file mode 100644
index 0000000..a81de58
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextScopeCondition.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.AtomicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+
+public class ContextScopeCondition extends AtomicCondition {
+
+ ContextScope contextScop;
+
+ public ContextScopeCondition(ContextScope contextScop) {
+ this.contextScop = contextScop;
+ }
+
+ public ContextScope getContextScop() {
+ return contextScop;
+ }
+
+ public void setContextScop(ContextScope contextScop) {
+ this.contextScop = contextScop;
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Equals;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextTypeCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextTypeCondition.java
new file mode 100644
index 0000000..dbd42f6
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextTypeCondition.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.AtomicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+
+public class ContextTypeCondition extends AtomicCondition {
+
+ ContextType contextType;
+
+ public ContextTypeCondition(ContextType contextType) {
+ this.contextType = contextType;
+ }
+
+ public ContextType getContextType() {
+ return contextType;
+ }
+
+ public void setContextType(ContextType contextType) {
+ this.contextType = contextType;
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Equals;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextValueTypeCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextValueTypeCondition.java
new file mode 100644
index 0000000..b6f330e
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/ContextValueTypeCondition.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.AtomicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+
+public class ContextValueTypeCondition extends AtomicCondition {
+
+ Class contextValueType;
+
+ public ContextValueTypeCondition(Class contextValueType) {
+ this.contextValueType = contextValueType;
+ }
+
+ public Class getContextValueType() {
+ return contextValueType;
+ }
+
+ public void setContextValueType(Class contextValueType) {
+ this.contextValueType = contextValueType;
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Equals;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/NearestCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/NearestCondition.java
new file mode 100644
index 0000000..43bd40d
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/NearestCondition.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+import com.webank.wedatasphere.linkis.cs.condition.UnaryLogicCondition;
+
+public class NearestCondition extends UnaryLogicCondition {
+
+ String currentNode;
+ Integer number;
+ Boolean upstreamOnly;
+
+ public NearestCondition(Condition origin, String currentNode, Integer number, Boolean upstreamOnly) {
+ super(origin);
+ this.currentNode = currentNode;
+ this.number = number;
+ this.upstreamOnly = upstreamOnly;
+ }
+
+ public String getCurrentNode() {
+ return currentNode;
+ }
+
+ public void setCurrentNode(String currentNode) {
+ this.currentNode = currentNode;
+ }
+
+ public Integer getNumber() {
+ return number;
+ }
+
+ public void setNumber(Integer number) {
+ this.number = number;
+ }
+
+ public Boolean getUpstreamOnly() {
+ return upstreamOnly;
+ }
+
+ public void setUpstreamOnly(Boolean upstreamOnly) {
+ this.upstreamOnly = upstreamOnly;
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Logic;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/NotCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/NotCondition.java
new file mode 100644
index 0000000..6635f32
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/NotCondition.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+import com.webank.wedatasphere.linkis.cs.condition.UnaryLogicCondition;
+
+public class NotCondition extends UnaryLogicCondition {
+ public NotCondition(Condition origin) {
+ super(origin);
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Logic;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/OrCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/OrCondition.java
new file mode 100644
index 0000000..0b0d4d8
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/OrCondition.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.condition.BinaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+
+public class OrCondition extends BinaryLogicCondition {
+ public OrCondition(Condition left, Condition right) {
+ super(left, right);
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Logic;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/RegexCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/RegexCondition.java
new file mode 100644
index 0000000..758a2df
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/condition/impl/RegexCondition.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.condition.impl;
+
+import com.webank.wedatasphere.linkis.cs.condition.AtomicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+
+public class RegexCondition extends AtomicCondition {
+
+ String regex;
+
+ public RegexCondition(String regex) {
+ this.regex = regex;
+ }
+
+ public String getRegex() {
+ return regex;
+ }
+
+ public void setRegex(String regex) {
+ this.regex = regex;
+ }
+
+ @Override
+ public ConditionType getConditionType() {
+ return ConditionType.Regex;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/exception/ContextSearchFailedException.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/exception/ContextSearchFailedException.java
new file mode 100644
index 0000000..83c095a
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/exception/ContextSearchFailedException.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.exception;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+
+public class ContextSearchFailedException extends ErrorException {
+ public ContextSearchFailedException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/AbstractConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/AbstractConditionExecution.java
new file mode 100644
index 0000000..5edd172
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/AbstractConditionExecution.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.ContextSearchRuler;
+import com.webank.wedatasphere.linkis.cs.optimize.ConditionOptimizer;
+import com.webank.wedatasphere.linkis.cs.optimize.OptimizedCondition;
+import com.webank.wedatasphere.linkis.cs.optimize.cost.ConditionCostCalculator;
+import com.webank.wedatasphere.linkis.cs.optimize.impl.CostBasedConditionOptimizer;
+
+import java.util.List;
+
+public abstract class AbstractConditionExecution implements ConditionExecution{
+
+ protected ContextSearchMatcher contextSearchMatcher;
+ protected ContextSearchRuler contextSearchRuler;
+ protected ContextCacheFetcher contextCacheFetcher;
+ protected ContextCacheService contextCacheService;
+ protected Condition condition;
+ protected ContextID contextID;
+
+ public AbstractConditionExecution(Condition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ this.condition = condition;
+ this.contextCacheService = contextCacheService;
+ this.contextID = contextID;
+ }
+
+ @Override
+ public List<ContextKeyValue> execute() {
+ if(needOptimization()){
+ OptimizedCondition optimizedCondition = getConditionOptimizer().optimize(condition);
+ }
+ ContextCacheFetcher fastFetcher = getFastFetcher();
+ if(fastFetcher != null){
+ return getContextSearchRuler().rule(fastFetcher.fetch(contextID));
+ } else {
+ return getContextCacheFetcher().fetch(contextID);
+ }
+ }
+
+ abstract protected boolean needOptimization();
+
+ abstract protected ContextCacheFetcher getFastFetcher();
+
+ @Override
+ public ContextSearchMatcher getContextSearchMatcher() {
+ return this.contextSearchMatcher;
+ }
+
+ @Override
+ public ContextSearchRuler getContextSearchRuler() {
+ return this.contextSearchRuler;
+ }
+
+ @Override
+ public ContextCacheFetcher getContextCacheFetcher() {
+ return this.contextCacheFetcher;
+ }
+
+ public ContextCacheService getContextCacheService() {
+ return contextCacheService;
+ }
+
+ public void setContextCacheService(ContextCacheService contextCacheService) {
+ this.contextCacheService = contextCacheService;
+ }
+
+ public Condition getCondition() {
+ return condition;
+ }
+
+ public ConditionOptimizer getConditionOptimizer() {
+ return new CostBasedConditionOptimizer(getConditionCostCalculator());
+ }
+
+ public ConditionCostCalculator getConditionCostCalculator(){
+ return new ConditionCostCalculator();
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ConditionExecution.java
new file mode 100644
index 0000000..781f0d2
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ConditionExecution.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.ContextSearchRuler;
+
+import java.util.List;
+
+public interface ConditionExecution {
+
+ ContextSearchMatcher getContextSearchMatcher();
+ ContextSearchRuler getContextSearchRuler();
+ ContextCacheFetcher getContextCacheFetcher();
+ List<ContextKeyValue> execute();
+ void setContextCacheService(ContextCacheService contextCacheService);
+ ContextCacheService getContextCacheService();
+ Condition getCondition();
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/AbstractContextCacheFetcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/AbstractContextCacheFetcher.java
new file mode 100644
index 0000000..b2e8f41
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/AbstractContextCacheFetcher.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.fetcher;
+
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+
+public abstract class AbstractContextCacheFetcher implements ContextCacheFetcher{
+
+ ContextCacheService contextCacheService;
+
+ public AbstractContextCacheFetcher(ContextCacheService contextCacheService) {
+ this.contextCacheService = contextCacheService;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/ContextCacheFetcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/ContextCacheFetcher.java
new file mode 100644
index 0000000..b2f3a3e
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/ContextCacheFetcher.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.fetcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+
+import java.util.List;
+
+public interface ContextCacheFetcher {
+
+ List<ContextKeyValue> fetch(ContextID contextID);
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/ContextTypeContextSearchFetcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/ContextTypeContextSearchFetcher.java
new file mode 100644
index 0000000..6e92731
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/ContextTypeContextSearchFetcher.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.fetcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class ContextTypeContextSearchFetcher extends AbstractContextCacheFetcher{
+
+ private static final Logger logger = LoggerFactory.getLogger(ContextTypeContextSearchFetcher.class);
+
+ ContextType contextType;
+
+ public ContextTypeContextSearchFetcher(ContextCacheService contextCacheService, ContextType contextType) {
+ super(contextCacheService);
+ this.contextType = contextType;
+ }
+
+ private ContextTypeContextSearchFetcher(ContextCacheService contextCacheService) {
+ super(contextCacheService);
+ }
+
+ @Override
+ public List<ContextKeyValue> fetch(ContextID contextID) {
+ return contextCacheService.getAllByType(contextID, contextType);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/IterateContextCacheFetcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/IterateContextCacheFetcher.java
new file mode 100644
index 0000000..513ac0e
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/fetcher/IterateContextCacheFetcher.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.fetcher;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.csid.ContextIDValue;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.ContextSearchRuler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class IterateContextCacheFetcher extends AbstractContextCacheFetcher{
+
+ private static final Logger logger = LoggerFactory.getLogger(IterateContextCacheFetcher.class);
+
+ ContextSearchRuler contextSearchRuler;
+
+ public IterateContextCacheFetcher(ContextCacheService contextCacheService, ContextSearchRuler contextSearchRuler) {
+ super(contextCacheService);
+ this.contextSearchRuler = contextSearchRuler;
+ }
+
+ private IterateContextCacheFetcher(ContextCacheService contextCacheService) {
+ super(contextCacheService);
+ }
+
+ @Override
+ public List<ContextKeyValue> fetch(ContextID contextID) {
+ return contextSearchRuler.rule(contextCacheService.getAll(contextID));
+ }
+
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/AndConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/AndConditionExecution.java
new file mode 100644
index 0000000..94154d4
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/AndConditionExecution.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.AndCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.RegexCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.AndLogicContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.RegexContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class AndConditionExecution extends BinaryLogicConditionExecution {
+
+ public AndConditionExecution(AndCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new AndLogicContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/BinaryLogicConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/BinaryLogicConditionExecution.java
new file mode 100644
index 0000000..069fcc9
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/BinaryLogicConditionExecution.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.BinaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextTypeContextSearchFetcher;
+
+public abstract class BinaryLogicConditionExecution extends AbstractConditionExecution {
+
+ ContextCacheFetcher fastFetcher;
+
+ public BinaryLogicConditionExecution(BinaryLogicCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ ContextTypeCondition contextTypeCondition = findFastCondition(condition.getLeft(), condition);
+ if(contextTypeCondition != null){
+ fastFetcher = new ContextTypeContextSearchFetcher(contextCacheService, contextTypeCondition.getContextType());
+ }
+ }
+
+ protected ContextTypeCondition findFastCondition(Condition condition, BinaryLogicCondition parent){
+ if(condition instanceof BinaryLogicCondition){
+ BinaryLogicCondition binaryLogicCondition = (BinaryLogicCondition) condition;
+ return findFastCondition(binaryLogicCondition.getLeft(), binaryLogicCondition);
+ } else if(condition instanceof ContextTypeCondition){
+ parent.setLeft(null);
+ return (ContextTypeCondition) condition;
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ protected ContextCacheFetcher getFastFetcher() {
+ return fastFetcher;
+ }
+
+ @Override
+ protected boolean needOptimization() {
+ return true;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContainsConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContainsConditionExecution.java
new file mode 100644
index 0000000..9961fd7
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContainsConditionExecution.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContainsCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContainsContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class ContainsConditionExecution extends AbstractConditionExecution {
+
+ public ContainsConditionExecution(ContainsCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new ContainsContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+
+ @Override
+ protected boolean needOptimization() {
+ return false;
+ }
+
+ @Override
+ protected ContextCacheFetcher getFastFetcher() {
+ return null;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextScopeConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextScopeConditionExecution.java
new file mode 100644
index 0000000..cd31fa7
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextScopeConditionExecution.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextScopeCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextScopeContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class ContextScopeConditionExecution extends AbstractConditionExecution {
+
+ public ContextScopeConditionExecution(ContextScopeCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new ContextScopeContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+
+ @Override
+ protected boolean needOptimization() {
+ return false;
+ }
+
+ @Override
+ protected ContextCacheFetcher getFastFetcher() {
+ return null;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextTypeConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextTypeConditionExecution.java
new file mode 100644
index 0000000..f486362
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextTypeConditionExecution.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextTypeContextSearchFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextTypeContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class ContextTypeConditionExecution extends AbstractConditionExecution {
+
+ public ContextTypeConditionExecution(ContextTypeCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new ContextTypeContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new ContextTypeContextSearchFetcher(contextCacheService, condition.getContextType());
+ }
+
+ @Override
+ protected boolean needOptimization() {
+ return false;
+ }
+
+ @Override
+ protected ContextCacheFetcher getFastFetcher() {
+ return null;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextValueTypeConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextValueTypeConditionExecution.java
new file mode 100644
index 0000000..753fb99
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/ContextValueTypeConditionExecution.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextValueTypeCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextTypeContextSearchFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextTypeContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextValueTypeContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class ContextValueTypeConditionExecution extends AbstractConditionExecution {
+
+ public ContextValueTypeConditionExecution(ContextValueTypeCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new ContextValueTypeContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+
+ @Override
+ protected boolean needOptimization() {
+ return false;
+ }
+
+ @Override
+ protected ContextCacheFetcher getFastFetcher() {
+ return null;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/NearestConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/NearestConditionExecution.java
new file mode 100644
index 0000000..4653849
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/NearestConditionExecution.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NearestCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NotCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.NearestLogicContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.NotLogicContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.SkipContextSearchMather;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.NearestContextSearchRuler;
+
+public class NearestConditionExecution extends UnaryLogicConditionExecution {
+
+ public NearestConditionExecution(NearestCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new NearestLogicContextSearchMatcher(condition);
+ this.contextSearchRuler = new NearestContextSearchRuler(contextSearchMatcher, contextCacheService, contextID, condition);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/NotConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/NotConditionExecution.java
new file mode 100644
index 0000000..17a979a
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/NotConditionExecution.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NotCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.OrCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.NotLogicContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.OrLogicContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class NotConditionExecution extends UnaryLogicConditionExecution {
+
+ public NotConditionExecution(NotCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new NotLogicContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/OrConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/OrConditionExecution.java
new file mode 100644
index 0000000..79afd87
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/OrConditionExecution.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.AndCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.OrCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.AndLogicContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.OrLogicContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class OrConditionExecution extends BinaryLogicConditionExecution {
+
+ public OrConditionExecution(OrCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new OrLogicContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/RegexConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/RegexConditionExecution.java
new file mode 100644
index 0000000..654f448
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/RegexConditionExecution.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.impl.RegexCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.IterateContextCacheFetcher;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.RegexContextSearchMatcher;
+import com.webank.wedatasphere.linkis.cs.execution.ruler.CommonListContextSearchRuler;
+
+public class RegexConditionExecution extends AbstractConditionExecution {
+
+ public RegexConditionExecution(RegexCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ this.contextSearchMatcher = new RegexContextSearchMatcher(condition);
+ this.contextSearchRuler = new CommonListContextSearchRuler(contextSearchMatcher);
+ this.contextCacheFetcher = new IterateContextCacheFetcher(contextCacheService, contextSearchRuler);
+ }
+
+ @Override
+ protected boolean needOptimization() {
+ return false;
+ }
+
+ @Override
+ protected ContextCacheFetcher getFastFetcher() {
+ return null;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/UnaryLogicConditionExecution.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/UnaryLogicConditionExecution.java
new file mode 100644
index 0000000..b52bafb
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/impl/UnaryLogicConditionExecution.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.condition.UnaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.contextcache.cache.ContextCache;
+import com.webank.wedatasphere.linkis.cs.execution.AbstractConditionExecution;
+import com.webank.wedatasphere.linkis.cs.execution.fetcher.ContextCacheFetcher;
+
+public abstract class UnaryLogicConditionExecution extends AbstractConditionExecution {
+
+ public UnaryLogicConditionExecution(UnaryLogicCondition condition, ContextCacheService contextCacheService, ContextID contextID) {
+ super(condition, contextCacheService, contextID);
+ }
+
+ @Override
+ protected ContextCacheFetcher getFastFetcher() {
+ return null;
+ }
+
+ @Override
+ protected boolean needOptimization() {
+ return true;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/AbstractContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/AbstractContextSearchMatcher.java
new file mode 100644
index 0000000..6b04176
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/AbstractContextSearchMatcher.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+
+public abstract class AbstractContextSearchMatcher implements ContextSearchMatcher{
+
+ Condition condition;
+
+ public AbstractContextSearchMatcher(Condition condition) {
+ this.condition = condition;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/AndLogicContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/AndLogicContextSearchMatcher.java
new file mode 100644
index 0000000..a94653d
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/AndLogicContextSearchMatcher.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.AndCondition;
+
+public class AndLogicContextSearchMatcher extends BinaryLogicContextSearchMatcher{
+
+ public AndLogicContextSearchMatcher(AndCondition condition) {
+ super(condition);
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return leftMatcher.match(contextKeyValue)
+ && rightMatcher.match(contextKeyValue);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/BinaryLogicContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/BinaryLogicContextSearchMatcher.java
new file mode 100644
index 0000000..45df78b
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/BinaryLogicContextSearchMatcher.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.BinaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+
+public abstract class BinaryLogicContextSearchMatcher extends AbstractContextSearchMatcher{
+
+ ContextSearchMatcher leftMatcher;
+ ContextSearchMatcher rightMatcher;
+
+ public BinaryLogicContextSearchMatcher(BinaryLogicCondition condition) {
+ super(condition);
+ this.leftMatcher = ConditionMatcherResolver.getMatcher(condition.getLeft());
+ this.rightMatcher = ConditionMatcherResolver.getMatcher(condition.getRight());
+ }
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ConditionMatcherResolver.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ConditionMatcherResolver.java
new file mode 100644
index 0000000..1610daa
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ConditionMatcherResolver.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.*;
+
+public class ConditionMatcherResolver {
+
+ public static ContextSearchMatcher getMatcher(Condition condition){
+ if(condition == null){
+ return new SkipContextSearchMather(condition);
+ }
+ if(condition instanceof ContextTypeCondition){
+ return new ContextTypeContextSearchMatcher((ContextTypeCondition) condition);
+ } else if(condition instanceof ContextScopeCondition){
+ return new ContextScopeContextSearchMatcher((ContextScopeCondition) condition);
+ } else if(condition instanceof RegexCondition){
+ return new RegexContextSearchMatcher((RegexCondition) condition);
+ } else if(condition instanceof ContainsCondition){
+ return new ContainsContextSearchMatcher((ContainsCondition) condition);
+ } else if(condition instanceof AndCondition){
+ return new AndLogicContextSearchMatcher((AndCondition) condition);
+ } else if(condition instanceof OrCondition){
+ return new OrLogicContextSearchMatcher((OrCondition) condition);
+ }else if(condition instanceof NearestCondition){
+ return new NearestLogicContextSearchMatcher((NearestCondition) condition);
+ }else if(condition instanceof ContextValueTypeCondition){
+ return new ContextValueTypeContextSearchMatcher((ContextValueTypeCondition) condition);
+ }
+ return new SkipContextSearchMather(condition);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContainsContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContainsContextSearchMatcher.java
new file mode 100644
index 0000000..838a6db
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContainsContextSearchMatcher.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContainsCondition;
+
+public class ContainsContextSearchMatcher extends AbstractContextSearchMatcher {
+
+ String value;
+
+ public ContainsContextSearchMatcher(ContainsCondition condition) {
+ super(condition);
+ this.value = condition.getValue();
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ if(contextKeyValue.getContextKey().getKeywords() == null){
+ return contextKeyValue.getContextKey().getKey().contains(value);
+ } else {
+ return contextKeyValue.getContextKey().getKey().contains(value)
+ || contextKeyValue.getContextKey().getKeywords().contains(value);
+ }
+
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextScopeContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextScopeContextSearchMatcher.java
new file mode 100644
index 0000000..0af1291
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextScopeContextSearchMatcher.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextScopeCondition;
+
+public class ContextScopeContextSearchMatcher extends AbstractContextSearchMatcher {
+
+ ContextScope contextScope;
+
+ public ContextScopeContextSearchMatcher(ContextScopeCondition condition) {
+ super(condition);
+ this.contextScope = condition.getContextScop();
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return contextScope.equals(contextKeyValue.getContextKey().getContextScope());
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextSearchMatcher.java
new file mode 100644
index 0000000..a6b1e76
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextSearchMatcher.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+
+public interface ContextSearchMatcher {
+
+ Boolean match(ContextKeyValue contextKeyValue);
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextTypeContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextTypeContextSearchMatcher.java
new file mode 100644
index 0000000..03f21c5
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextTypeContextSearchMatcher.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+
+public class ContextTypeContextSearchMatcher extends AbstractContextSearchMatcher{
+
+ ContextType contextType;
+
+ public ContextTypeContextSearchMatcher(ContextTypeCondition condition) {
+ super(condition);
+ contextType = condition.getContextType();
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return contextType.equals(contextKeyValue.getContextKey().getContextType());
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextValueTypeContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextValueTypeContextSearchMatcher.java
new file mode 100644
index 0000000..0713f59
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/ContextValueTypeContextSearchMatcher.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextTypeCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextValueTypeCondition;
+import org.reflections.ReflectionUtils;
+
+public class ContextValueTypeContextSearchMatcher extends AbstractContextSearchMatcher{
+
+ Class contextValueType;
+
+ public ContextValueTypeContextSearchMatcher(ContextValueTypeCondition condition) {
+ super(condition);
+ contextValueType = condition.getContextValueType();
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return contextValueType.isInstance(contextKeyValue.getContextValue().getValue());
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/NearestLogicContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/NearestLogicContextSearchMatcher.java
new file mode 100644
index 0000000..095ed12
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/NearestLogicContextSearchMatcher.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NearestCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NotCondition;
+
+public class NearestLogicContextSearchMatcher extends UnaryLogicContextSearchMatcher{
+
+ public NearestLogicContextSearchMatcher(NearestCondition condition) {
+ super(condition);
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return originalMatcher.match(contextKeyValue);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/NotLogicContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/NotLogicContextSearchMatcher.java
new file mode 100644
index 0000000..87ac134
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/NotLogicContextSearchMatcher.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.BinaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NotCondition;
+
+public class NotLogicContextSearchMatcher extends UnaryLogicContextSearchMatcher{
+
+ public NotLogicContextSearchMatcher(NotCondition condition) {
+ super(condition);
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return !originalMatcher.match(contextKeyValue);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/OrLogicContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/OrLogicContextSearchMatcher.java
new file mode 100644
index 0000000..3c05931
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/OrLogicContextSearchMatcher.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.BinaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.OrCondition;
+
+public class OrLogicContextSearchMatcher extends BinaryLogicContextSearchMatcher{
+
+ public OrLogicContextSearchMatcher(OrCondition condition) {
+ super(condition);
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return leftMatcher.match(contextKeyValue)
+ || rightMatcher.match(contextKeyValue);
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/RegexContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/RegexContextSearchMatcher.java
new file mode 100644
index 0000000..0f27d7b
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/RegexContextSearchMatcher.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.RegexCondition;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RegexContextSearchMatcher extends AbstractContextSearchMatcher{
+
+ Pattern pattern;
+
+ public RegexContextSearchMatcher(RegexCondition condition) {
+ super(condition);
+ pattern = Pattern.compile(condition.getRegex());
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ Matcher keyMatcher = pattern.matcher(contextKeyValue.getContextKey().getKey());
+ if(contextKeyValue.getContextKey().getKeywords() == null){
+ return keyMatcher.find();
+ } else {
+ Matcher keywordsMatcher = pattern.matcher(contextKeyValue.getContextKey().getKeywords());
+ return keyMatcher.find() || keywordsMatcher.find();
+ }
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/SkipContextSearchMather.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/SkipContextSearchMather.java
new file mode 100644
index 0000000..d9eb9f0
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/SkipContextSearchMather.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+
+public class SkipContextSearchMather extends AbstractContextSearchMatcher{
+
+ public SkipContextSearchMather(Condition condition) {
+ super(condition);
+ }
+
+ @Override
+ public Boolean match(ContextKeyValue contextKeyValue) {
+ return true;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/UnaryLogicContextSearchMatcher.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/UnaryLogicContextSearchMatcher.java
new file mode 100644
index 0000000..9ddf5a7
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/matcher/UnaryLogicContextSearchMatcher.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.matcher;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.UnaryLogicCondition;
+
+public abstract class UnaryLogicContextSearchMatcher extends AbstractContextSearchMatcher{
+
+ ContextSearchMatcher originalMatcher;
+
+ public UnaryLogicContextSearchMatcher(UnaryLogicCondition condition) {
+ super(condition);
+ originalMatcher = ConditionMatcherResolver.getMatcher(condition.getOrigin());
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/AbstractContextSearchRuler.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/AbstractContextSearchRuler.java
new file mode 100644
index 0000000..f67ceb2
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/AbstractContextSearchRuler.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.ruler;
+
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextSearchMatcher;
+
+public abstract class AbstractContextSearchRuler implements ContextSearchRuler{
+
+ ContextSearchMatcher matcher;
+
+ public AbstractContextSearchRuler(ContextSearchMatcher matcher) {
+ this.matcher = matcher;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/CommonListContextSearchRuler.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/CommonListContextSearchRuler.java
new file mode 100644
index 0000000..b5c7e66
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/CommonListContextSearchRuler.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.ruler;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextSearchMatcher;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class CommonListContextSearchRuler extends AbstractContextSearchRuler{
+
+ private static Logger logger = LoggerFactory.getLogger(CommonListContextSearchRuler.class);
+
+ public CommonListContextSearchRuler(ContextSearchMatcher matcher) {
+ super(matcher);
+ }
+
+ @Override
+ public List<ContextKeyValue> rule(List<ContextKeyValue> contextKeyValues) {
+ List<ContextKeyValue> filtered = Lists.newArrayList();
+ if(CollectionUtils.isEmpty(contextKeyValues)){
+ logger.warn("Empty result fetched from cache.");
+ return filtered;
+ }
+ int size = contextKeyValues.size();
+ for(int i=0;i<size;i++){
+ ContextKeyValue contextKeyValue = contextKeyValues.get(i);
+ if(matcher.match(contextKeyValue)){
+ filtered.add(contextKeyValue);
+ }
+ }
+ return filtered;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/ContextSearchRuler.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/ContextSearchRuler.java
new file mode 100644
index 0000000..614a30a
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/ContextSearchRuler.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.ruler;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+
+import java.util.List;
+
+public interface ContextSearchRuler {
+
+ List<ContextKeyValue> rule(List<ContextKeyValue> contextKeyValues);
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/NearestContextSearchRuler.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/NearestContextSearchRuler.java
new file mode 100644
index 0000000..4bd46d9
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/execution/ruler/NearestContextSearchRuler.java
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.execution.ruler;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import com.webank.wedatasphere.linkis.cs.condition.impl.NearestCondition;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.execution.matcher.ContextSearchMatcher;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+public class NearestContextSearchRuler extends AbstractContextSearchRuler{
+
+ private static Logger logger = LoggerFactory.getLogger(NearestContextSearchRuler.class);
+
+ static int LARGE_INT = 100000;
+ static int SMALL_INT = -100000;
+ static String DOT = ".";
+
+ ContextCacheService contextCacheService;
+ ContextID contextID;
+ NearestCondition nearestCondition;
+ CommonListContextSearchRuler commonListContextSearchRuler;
+
+
+ public NearestContextSearchRuler(ContextSearchMatcher matcher, ContextCacheService contextCacheService, ContextID contextID, NearestCondition condition) {
+ super(matcher);
+ this.contextCacheService = contextCacheService;
+ this.contextID = contextID;
+ this.nearestCondition = condition;
+ this.commonListContextSearchRuler = new CommonListContextSearchRuler(matcher);
+ }
+
+ @Override
+ public List<ContextKeyValue> rule(List<ContextKeyValue> contextKeyValues) {
+ contextKeyValues = commonListContextSearchRuler.rule(contextKeyValues);
+ List<ContextKeyValue> filtered = Lists.newArrayList();
+ ContextKey flowInfosKey = new CommonContextKey();
+ flowInfosKey.setContextType(ContextType.OBJECT);
+ flowInfosKey.setKey(CSCommonUtils.FLOW_INFOS);
+ ContextKeyValue flowInfos = contextCacheService.get(contextID, flowInfosKey);
+
+ if(null != flowInfos && flowInfos.getContextValue().getValue() instanceof CSFlowInfos){
+ CSFlowInfos csFlowInfos = (CSFlowInfos) flowInfos.getContextValue().getValue();
+ logger.info("Calculate nearest nodes based on flow info: \n" + BDPJettyServerHelper.gson().toJson(csFlowInfos));
+ List<Map<String, String>> edges = (List<Map<String, String>>) csFlowInfos.getInfos().get("edges");
+ Map<String, String> idNodeName = (Map<String, String>) csFlowInfos.getInfos().get(CSCommonUtils.ID_NODE_NAME);
+ String[] fullNodeName = StringUtils.split(nearestCondition.getCurrentNode(), DOT);
+ String currentNodeName = fullNodeName[fullNodeName.length - 1];
+ List<ContextKeyValue> notUpstream = Lists.newArrayList();
+
+ contextKeyValues.sort(new Comparator<ContextKeyValue>() {
+ @Override
+ public int compare(ContextKeyValue o1, ContextKeyValue o2) {
+ return distance(o1) - distance(o2);
+ }
+
+ private int distance(ContextKeyValue contextKeyValue){
+ String nodeNameToCalc = StringUtils.substringBetween(contextKeyValue.getContextKey().getKey(), CSCommonUtils.NODE_PREFIX, DOT);
+ if(StringUtils.isBlank(nodeNameToCalc)){
+ return LARGE_INT;
+ }
+ if(nodeNameToCalc.equals(currentNodeName)){
+ return 0;
+ }
+ int upDistance = upstreamDistance(nodeNameToCalc);
+ if(nearestCondition.getUpstreamOnly()){
+ if(upDistance >= LARGE_INT){
+ notUpstream.add(contextKeyValue);
+ }
+ return upDistance;
+ } else {
+ int downDistance = downstreamDistance(nodeNameToCalc);
+ return Integer.min(upDistance, downDistance);
+ }
+ }
+
+ private int upstreamDistance(String nodeNameToCalc){
+ int minDistance = LARGE_INT;
+ for(Map<String, String> edge : edges) {
+ if(getNodeName(edge.get("source")).equals(nodeNameToCalc)){
+ if(getNodeName(edge.get("target")).equals(currentNodeName)){
+ return 1;
+ } else {
+ minDistance = Integer.min(minDistance,upstreamDistance(getNodeName(edge.get("target"))) + 1);
+ }
+ }
+ }
+ return minDistance;
+ }
+
+ private int downstreamDistance(String nodeNameToCalc){
+ int minDistance = LARGE_INT;
+ for(Map<String, String> edge : edges) {
+ if(getNodeName(edge.get("target")).equals(nodeNameToCalc)){
+ if(getNodeName(edge.get("source")).equals(currentNodeName)){
+ return 1;
+ } else {
+ minDistance = downstreamDistance(getNodeName(edge.get("source"))) + 1;
+ }
+ }
+ }
+ return minDistance;
+ }
+
+ private String getNodeName(String nodeId){
+ if(StringUtils.isBlank(nodeId) || null == idNodeName){
+ return "";
+ }
+ return idNodeName.get(nodeId);
+ }
+ });
+ if(nearestCondition.getUpstreamOnly()){
+ contextKeyValues.removeAll(notUpstream);
+ }
+ int endIndex = nearestCondition.getNumber() < contextKeyValues.size() ? nearestCondition.getNumber() : contextKeyValues.size();
+ filtered = contextKeyValues.subList(0, endIndex);
+ }
+
+ return filtered;
+ }
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/ConditionOptimizer.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/ConditionOptimizer.java
new file mode 100644
index 0000000..f594006
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/ConditionOptimizer.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.optimize;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+
+public interface ConditionOptimizer {
+
+ public OptimizedCondition optimize(Condition condition);
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/OptimizedCondition.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/OptimizedCondition.java
new file mode 100644
index 0000000..54f4bf7
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/OptimizedCondition.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.optimize;
+
+import com.webank.wedatasphere.linkis.cs.condition.BinaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.UnaryLogicCondition;
+import com.webank.wedatasphere.linkis.cs.optimize.cost.ConditionCostCalculator;
+import com.webank.wedatasphere.linkis.cs.optimize.dfs.Node;
+
+public class OptimizedCondition implements Node {
+
+ static Double HIGH_PRIORITY = 1d;
+ static Double LOW_PRIORITY = 0.5d;
+
+ Condition condition;
+ Double cost;
+ Double priority;
+ boolean visited = false;
+ OptimizedCondition left;
+ OptimizedCondition right;
+
+ public OptimizedCondition(Condition condition, ConditionCostCalculator conditionCostCalculator) {
+ new OptimizedCondition(condition, HIGH_PRIORITY, conditionCostCalculator);
+ }
+
+ public OptimizedCondition(Condition condition, Double priority, ConditionCostCalculator conditionCostCalculator) {
+ this.condition = condition;
+ this.priority = priority;
+ this.cost = conditionCostCalculator.calculate(condition);
+ if(condition instanceof BinaryLogicCondition){
+ BinaryLogicCondition binaryLogicCondition = (BinaryLogicCondition) condition;
+ this.left = new OptimizedCondition(binaryLogicCondition.getLeft(), HIGH_PRIORITY, conditionCostCalculator);
+ this.right = new OptimizedCondition(binaryLogicCondition.getRight(), LOW_PRIORITY, conditionCostCalculator);
+ } else if(condition instanceof UnaryLogicCondition){
+ this.left = new OptimizedCondition(((UnaryLogicCondition) condition).getOrigin(), conditionCostCalculator);
+ }
+ }
+
+ public Condition getCondition() {
+ return condition;
+ }
+
+ public void setCondition(Condition condition) {
+ this.condition = condition;
+ }
+
+ @Override
+ public Double getCost() {
+ return this.cost;
+ }
+
+ @Override
+ public Double getPriority() {
+ return this.priority;
+ }
+
+ @Override
+ public Node getLeft() {
+ return this.left;
+ }
+
+ @Override
+ public Node getRight() {
+ return this.right;
+ }
+
+ @Override
+ public void shift() {
+ if(condition instanceof BinaryLogicCondition){
+ OptimizedCondition tmp = this.left;
+ this.left = this.right;
+ this.right = tmp;
+ this.right.priority = this.left.priority;
+ this.left.priority = tmp.priority;
+ ((BinaryLogicCondition) condition).shift();
+ }
+ }
+
+ @Override
+ public boolean visited() {
+ return visited;
+ }
+
+ @Override
+ public void visit() {
+ this.visited = true;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/cost/ConditionCostCalculator.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/cost/ConditionCostCalculator.java
new file mode 100644
index 0000000..8c7f431
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/cost/ConditionCostCalculator.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.optimize.cost;
+
+import com.google.common.collect.Maps;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.condition.AtomicCondition;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContainsCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.ContextValueTypeCondition;
+import com.webank.wedatasphere.linkis.cs.condition.impl.RegexCondition;
+
+import java.util.Map;
+
+public class ConditionCostCalculator {
+
+ static Map<Class, Double> initialCost = Maps.newHashMap();
+
+ static{
+ initialCost.put(RegexCondition.class, 100d);
+ initialCost.put(ContainsCondition.class, 10d);
+ initialCost.put(ContextScope.class, 10d);
+ initialCost.put(ContextType.class, 1d);
+ initialCost.put(ContextValueTypeCondition.class, 1d);
+ }
+
+ public Double calculate(Condition condition){
+ if(condition instanceof AtomicCondition){
+ return initialCost.get(condition.getClass());
+ }
+ return 0d;
+ }
+
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/BinaryTree.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/BinaryTree.java
new file mode 100644
index 0000000..ebae26e
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/BinaryTree.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.optimize.dfs;
+
+public interface BinaryTree {
+ BinaryTree getLeft();
+ BinaryTree getRight();
+ Node getRootNode();
+ Double getCost();
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/MinCostBinaryTree.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/MinCostBinaryTree.java
new file mode 100644
index 0000000..9d8cea1
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/MinCostBinaryTree.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.optimize.dfs;
+
+public class MinCostBinaryTree implements BinaryTree{
+
+ Node rootNode;
+ BinaryTree left;
+ BinaryTree right;
+ Double cost;
+
+ public MinCostBinaryTree(Node rootNode) {
+ this.rootNode = rootNode;
+ if(this.rootNode.visited()){
+ return;
+ }
+ if(rootNode.getLeft() == null){
+ this.cost = rootNode.getCost();
+ return;
+ }
+ if(rootNode.getRight() == null){
+ this.left = new MinCostBinaryTree(rootNode.getLeft());
+ this.cost = this.left.getCost();
+ return;
+ }
+
+ BinaryTree firstLeft = new MinCostBinaryTree(rootNode.getLeft());
+ BinaryTree firstRight = new MinCostBinaryTree(rootNode.getRight());
+ Double firstCost = firstLeft.getCost() * rootNode.getLeft().getPriority() + firstRight.getCost()*rootNode.getRight().getPriority();
+
+ rootNode.shift();
+
+ BinaryTree secondLeft = new MinCostBinaryTree(rootNode.getLeft());
+ BinaryTree secondRight = new MinCostBinaryTree(rootNode.getRight());
+ Double secondCost = secondLeft.getCost() * rootNode.getLeft().getPriority() + secondRight.getCost()*rootNode.getRight().getPriority();
+
+ if(firstCost > secondCost){
+ this.left = secondLeft;
+ this.right = secondRight;
+ this.cost = secondCost;
+ } else {
+ rootNode.shift();
+ this.left = firstLeft;
+ this.right = secondRight;
+ this.cost = firstCost;
+ }
+ this.rootNode.visit();
+ }
+
+ @Override
+ public BinaryTree getLeft() {
+ return this.left;
+ }
+
+ @Override
+ public BinaryTree getRight() {
+ return this.right;
+ }
+
+ @Override
+ public Node getRootNode() {
+ return this.rootNode;
+ }
+
+ @Override
+ public Double getCost() {
+ return this.cost;
+ }
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/Node.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/Node.java
new file mode 100644
index 0000000..85fb8ff
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/dfs/Node.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.optimize.dfs;
+
+public interface Node {
+ Double getCost();
+ Double getPriority();
+ Node getLeft();
+ Node getRight();
+ void shift();
+ boolean visited();
+ void visit();
+}
diff --git a/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/impl/CostBasedConditionOptimizer.java b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/impl/CostBasedConditionOptimizer.java
new file mode 100644
index 0000000..ba8fb4a
--- /dev/null
+++ b/contextservice/cs-search/src/main/java/com/webank/wedatasphere/linkis/cs/optimize/impl/CostBasedConditionOptimizer.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.optimize.impl;
+
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.optimize.ConditionOptimizer;
+import com.webank.wedatasphere.linkis.cs.optimize.OptimizedCondition;
+import com.webank.wedatasphere.linkis.cs.optimize.cost.ConditionCostCalculator;
+import com.webank.wedatasphere.linkis.cs.optimize.dfs.MinCostBinaryTree;
+
+public class CostBasedConditionOptimizer implements ConditionOptimizer {
+
+ ConditionCostCalculator conditionCostCalculator;
+
+ public CostBasedConditionOptimizer(ConditionCostCalculator conditionCostCalculator) {
+ this.conditionCostCalculator = conditionCostCalculator;
+ }
+
+ @Override
+ public OptimizedCondition optimize(Condition condition) {
+ OptimizedCondition dfsTreeNode = new OptimizedCondition(condition, conditionCostCalculator);
+ MinCostBinaryTree minCostBinaryTree = new MinCostBinaryTree(dfsTreeNode);
+ return dfsTreeNode;
+ }
+
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/AndTest.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/AndTest.java
new file mode 100644
index 0000000..8611ad0
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/AndTest.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+public class AndTest {
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContainsTest.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContainsTest.java
new file mode 100644
index 0000000..98cebf4
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContainsTest.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.construction.ConditionBuilder;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.csid.TestContextID;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKey;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKeyValue;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+
+public class ContainsTest {
+ List<ContextKeyValue> contextKeyValues = Lists.newArrayList();
+
+ @Before
+ public void setUp() throws Exception {
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("abc345efgabcab");
+ contextKey1.setContextType(ContextType.DATA);
+ contextKey1.setContextScope(ContextScope.PRIVATE);
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue1 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey1);
+ contextKeyValue1.setContextValue(null);
+ contextKeyValues.add(contextKeyValue1);
+
+ ContextID contextID2 = new TestContextID();
+ contextID2.setContextId("id");
+ ContextKey contextKey2 = new TestContextKey();
+ contextKey2.setKey("2342342342342");
+ contextKey2.setContextType(ContextType.METADATA);
+ contextKey2.setContextScope(ContextScope.PROTECTED);
+ contextKey2.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue2 = new TestContextKeyValue();
+ contextKeyValue2.setContextKey(contextKey2);
+ contextKeyValue2.setContextValue(null);
+ contextKeyValues.add(contextKeyValue2);
+
+ ContextID contextID3 = new TestContextID();
+ contextID3.setContextId("id");
+ ContextKey contextKey3 = new TestContextKey();
+ contextKey3.setKey("34646456e");
+ contextKey3.setContextType(ContextType.COST);
+ contextKey3.setContextScope(ContextScope.PROTECTED);
+ contextKey3.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue3 = new TestContextKeyValue();
+ contextKeyValue3.setContextKey(contextKey3);
+ contextKeyValue3.setContextValue(null);
+ contextKeyValues.add(contextKeyValue3);
+ }
+
+ @Test
+ public void testSearch() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAll(Mockito.any(ContextID.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder.contains("abc");
+ Condition condition = conditionBuilder.build();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(1, list.size());
+ }
+
+ @Test
+ public void testSearchNegate() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAll(Mockito.any(ContextID.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder.contains("abc");
+ Condition condition = conditionBuilder.build().not();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(2, list.size());
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextScopeTest.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextScopeTest.java
new file mode 100644
index 0000000..d8c4f9c
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextScopeTest.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.construction.ConditionBuilder;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.csid.TestContextID;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKey;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKeyValue;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+
+public class ContextScopeTest {
+
+ List<ContextKeyValue> contextKeyValues = Lists.newArrayList();
+
+ @Before
+ public void setUp() throws Exception {
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("key1");
+ contextKey1.setContextType(ContextType.DATA);
+ contextKey1.setContextScope(ContextScope.PRIVATE);
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue1 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey1);
+ contextKeyValue1.setContextValue(null);
+ contextKeyValues.add(contextKeyValue1);
+
+ ContextID contextID2 = new TestContextID();
+ contextID2.setContextId("id");
+ ContextKey contextKey2 = new TestContextKey();
+ contextKey2.setKey("key2");
+ contextKey2.setContextType(ContextType.METADATA);
+ contextKey2.setContextScope(ContextScope.PROTECTED);
+ contextKey2.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue2 = new TestContextKeyValue();
+ contextKeyValue2.setContextKey(contextKey2);
+ contextKeyValue2.setContextValue(null);
+ contextKeyValues.add(contextKeyValue2);
+
+ ContextID contextID3 = new TestContextID();
+ contextID3.setContextId("id");
+ ContextKey contextKey3 = new TestContextKey();
+ contextKey3.setKey("key3");
+ contextKey3.setContextType(ContextType.COST);
+ contextKey3.setContextScope(ContextScope.PROTECTED);
+ contextKey3.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue3 = new TestContextKeyValue();
+ contextKeyValue3.setContextKey(contextKey3);
+ contextKeyValue3.setContextValue(null);
+ contextKeyValues.add(contextKeyValue3);
+ }
+
+ @Test
+ public void testSearch() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAll(Mockito.any(ContextID.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder
+ .contextScopes(Lists.newArrayList(ContextScope.PROTECTED));
+ Condition condition = conditionBuilder.build();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(2, list.size());
+ }
+
+ @Test
+ public void testSearchNegate() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAll(Mockito.any(ContextID.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder
+ .contextScopes(Lists.newArrayList(ContextScope.PROTECTED));
+ Condition condition = conditionBuilder.build().not();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(1, list.size());
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextSearchTest.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextSearchTest.java
new file mode 100644
index 0000000..334b3c2
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextSearchTest.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.construction.ConditionBuilder;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.csid.TestContextID;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKey;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKeyValue;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+
+public class ContextSearchTest {
+
+ List<ContextKeyValue> contextKeyValues = Lists.newArrayList();
+
+ @Before
+ public void setUp() throws Exception {
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("key1");
+ contextKey1.setContextType(ContextType.DATA);
+ contextKey1.setContextScope(ContextScope.PRIVATE);
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue1 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey1);
+ contextKeyValue1.setContextValue(null);
+ contextKeyValues.add(contextKeyValue1);
+
+ ContextID contextID2 = new TestContextID();
+ contextID2.setContextId("id");
+ ContextKey contextKey2 = new TestContextKey();
+ contextKey2.setKey("key2");
+ contextKey2.setContextType(ContextType.METADATA);
+ contextKey2.setContextScope(ContextScope.PROTECTED);
+ contextKey2.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue2 = new TestContextKeyValue();
+ contextKeyValue2.setContextKey(contextKey2);
+ contextKeyValue2.setContextValue(null);
+ contextKeyValues.add(contextKeyValue2);
+ }
+
+ @Test
+ public void testSearch() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAll(Mockito.any(ContextID.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder
+ .contextScopes(Lists.newArrayList(ContextScope.PRIVATE))
+ .contains("key");
+ Condition condition = conditionBuilder.build();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(1, list.size());
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextTypeTest.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextTypeTest.java
new file mode 100644
index 0000000..de1d2e3
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/ContextTypeTest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.construction.ConditionBuilder;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.csid.TestContextID;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKey;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKeyValue;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+
+public class ContextTypeTest {
+
+ List<ContextKeyValue> contextKeyValues = Lists.newArrayList();
+
+ @Before
+ public void setUp() throws Exception {
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("key1");
+ contextKey1.setContextType(ContextType.DATA);
+ contextKey1.setContextScope(ContextScope.PRIVATE);
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue1 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey1);
+ contextKeyValue1.setContextValue(null);
+ contextKeyValues.add(contextKeyValue1);
+
+ ContextID contextID2 = new TestContextID();
+ contextID2.setContextId("id");
+ ContextKey contextKey2 = new TestContextKey();
+ contextKey2.setKey("key2");
+ contextKey2.setContextType(ContextType.METADATA);
+ contextKey2.setContextScope(ContextScope.PROTECTED);
+ contextKey2.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue2 = new TestContextKeyValue();
+ contextKeyValue2.setContextKey(contextKey2);
+ contextKeyValue2.setContextValue(null);
+ //contextKeyValues.add(contextKeyValue2);
+ }
+
+ @Test
+ public void testSearch() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAllByType(Mockito.any(ContextID.class), Mockito.any(ContextType.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder.contextTypes(Lists.newArrayList(ContextType.DATA));
+ Condition condition = conditionBuilder.build();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(1, list.size());
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/RegexTest.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/RegexTest.java
new file mode 100644
index 0000000..64aea84
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/RegexTest.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs;
+
+import com.google.common.collect.Lists;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.construction.ConditionBuilder;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.csid.TestContextID;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKey;
+import com.webank.wedatasphere.linkis.cs.keyword.TestContextKeyValue;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+
+public class RegexTest {
+
+ List<ContextKeyValue> contextKeyValues = Lists.newArrayList();
+
+ @Before
+ public void setUp() throws Exception {
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ ContextKey contextKey1 = new TestContextKey();
+ contextKey1.setKey("abc345efgabcab");
+ contextKey1.setContextType(ContextType.DATA);
+ contextKey1.setContextScope(ContextScope.PRIVATE);
+ contextKey1.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue1 = new TestContextKeyValue();
+ contextKeyValue1.setContextKey(contextKey1);
+ contextKeyValue1.setContextValue(null);
+ contextKeyValues.add(contextKeyValue1);
+
+ ContextID contextID2 = new TestContextID();
+ contextID2.setContextId("id");
+ ContextKey contextKey2 = new TestContextKey();
+ contextKey2.setKey("2342342342342");
+ contextKey2.setContextType(ContextType.METADATA);
+ contextKey2.setContextScope(ContextScope.PROTECTED);
+ contextKey2.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue2 = new TestContextKeyValue();
+ contextKeyValue2.setContextKey(contextKey2);
+ contextKeyValue2.setContextValue(null);
+ contextKeyValues.add(contextKeyValue2);
+
+ ContextID contextID3 = new TestContextID();
+ contextID3.setContextId("id");
+ ContextKey contextKey3 = new TestContextKey();
+ contextKey3.setKey("34646456e");
+ contextKey3.setContextType(ContextType.COST);
+ contextKey3.setContextScope(ContextScope.PROTECTED);
+ contextKey3.setKeywords("keyword1,keyword2,keyword3");
+ ContextKeyValue contextKeyValue3 = new TestContextKeyValue();
+ contextKeyValue3.setContextKey(contextKey3);
+ contextKeyValue3.setContextValue(null);
+ contextKeyValues.add(contextKeyValue3);
+ }
+
+ @Test
+ public void testSearch() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAll(Mockito.any(ContextID.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder.regex("[abc]");
+ Condition condition = conditionBuilder.build();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(1, list.size());
+ }
+
+ @Test
+ public void testSearchNegate() throws ContextSearchFailedException {
+
+ ContextSearch contextSearch = new DefaultContextSearch();
+ ContextCacheService contextCacheService = Mockito.mock(ContextCacheService.class);
+ Mockito.when(contextCacheService.getAll(Mockito.any(ContextID.class))).thenReturn(contextKeyValues);
+
+ ConditionBuilder conditionBuilder = ConditionBuilder.newBuilder();
+ conditionBuilder.regex("[abc]");
+ Condition condition = conditionBuilder.build().not();
+
+ ContextID contextID = new TestContextID();
+ contextID.setContextId("id");
+ List<ContextKeyValue> list = contextSearch.search(contextCacheService, contextID, condition);
+ Assert.assertEquals(2, list.size());
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/csid/TestContextID.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/csid/TestContextID.java
new file mode 100644
index 0000000..0fbdb6b
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/csid/TestContextID.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.csid;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 20:41
+ */
+public class TestContextID implements ContextID {
+
+ String contextID;
+
+ @Override
+ public String getContextId() {
+ return contextID;
+ }
+
+ @Override
+ public void setContextId(String contextId) {
+ this.contextID = contextId;
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextKey.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextKey.java
new file mode 100644
index 0000000..74bd9c6
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextKey.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.keyword;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 16:32
+ */
+public class TestContextKey implements ContextKey {
+
+ private String key;
+
+ private String keywords;
+
+ private ContextScope contextScope;
+
+ private ContextType contextType;
+
+ @KeywordMethod
+ @Override
+ public String getKey() {
+ return this.key;
+ }
+
+ @Override
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public ContextType getContextType() {
+ return contextType;
+ }
+
+ @Override
+ public void setContextType(ContextType contextType) {
+ this.contextType = contextType;
+ }
+
+ @Override
+ public ContextScope getContextScope() {
+ return contextScope;
+ }
+
+ @Override
+ public void setContextScope(ContextScope contextScope) {
+ this.contextScope = contextScope;
+ }
+
+ @KeywordMethod(splitter = ",")
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+
+
+ @Override
+ public int getType() {
+ return 0;
+ }
+
+ @Override
+ public void setType(int type) {
+
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextKeyValue.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextKeyValue.java
new file mode 100644
index 0000000..79d8a67
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextKeyValue.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.keyword;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 16:46
+ */
+public class TestContextKeyValue implements ContextKeyValue {
+
+ private ContextKey contextKey;
+
+ private ContextValue contextValue;
+
+ @Override
+ public ContextKey getContextKey() {
+ return this.contextKey;
+ }
+
+ @Override
+ public void setContextKey(ContextKey contextKey) {
+ this.contextKey = contextKey;
+ }
+
+ @Override
+ public ContextValue getContextValue() {
+ return this.contextValue;
+ }
+
+ @Override
+ public void setContextValue(ContextValue contextValue) {
+ this.contextValue = contextValue;
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextValue.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextValue.java
new file mode 100644
index 0000000..0a34ed4
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/keyword/TestContextValue.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.keyword;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ValueBean;
+
+/**
+ * @author peacewong
+ * @date 2020/2/13 16:44
+ */
+public class TestContextValue implements ContextValue {
+
+ private Object value;
+
+ private String keywords;
+
+ @KeywordMethod(splitter = "-")
+ @Override
+ public String getKeywords() {
+ return this.keywords;
+ }
+
+ @Override
+ public void setKeywords(String keywords) {
+ this.keywords = keywords;
+ }
+
+ @KeywordMethod(regex = "hello")
+ @Override
+ public Object getValue() {
+ return this.value;
+ }
+
+ @Override
+ public void setValue(Object value) {
+ this.value = value;
+ }
+}
diff --git a/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/parser/ApiJsonTest.java b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/parser/ApiJsonTest.java
new file mode 100644
index 0000000..7f3928b
--- /dev/null
+++ b/contextservice/cs-search/src/test/java/com/webank/wedatasphere/linkis/cs/parser/ApiJsonTest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.parser;
+
+import com.google.common.collect.Sets;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.condition.ConditionType;
+import com.webank.wedatasphere.linkis.cs.condition.construction.ConditionParser;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public class ApiJsonTest {
+
+ @Test
+ public void test() throws IOException {
+ String apiString = "{\n" +
+ "\t\"type\":\"And\",\n" +
+ "\t\"left\":{\n" +
+ "\t\t\"type\":\"ContextType\",\n" +
+ "\t\t\"contextType\":\"DATA\"\n" +
+ "\t},\n" +
+ "\t\"right\":{\n" +
+ "\t\t\"type\":\"And\",\n" +
+ "\t\t\"left\":{\n" +
+ "\t\t\t\"type\":\"ContextScope\",\n" +
+ "\t\t\t\"contextScope\":\"PRIVATE\"\n" +
+ "\t\t},\n" +
+ "\t\t\"right\":{\n" +
+ "\t\t\t\"type\":\"Regex\",\n" +
+ "\t\t\t\"regex\":\"[abc]]\"\n" +
+ "\t\t}\n" +
+ "\t}\n" +
+ "}";
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode jsonNode = objectMapper.readTree(apiString);
+ Map<Object, Object> conditionMap = objectMapper.convertValue(jsonNode, new TypeReference<Map<Object, Object>>(){});
+ ConditionParser conditionParser = ConditionParser.parserMap.get(conditionMap.get("type"));
+ Condition condition = conditionParser.parse(conditionMap);
+ Assert.assertEquals(condition.getConditionType(), ConditionType.Logic);
+
+ }
+
+ @Test
+ public void temp() {
+ String test = "{\"cols\":[{\"name\":\"birthday\",\"visualType\":\"string\",\"type\":\"category\",\"config\":true,\"field\":{\"alias\":\"\",\"desc\":\"\",\"useExpression\":false},\"format\":{\"formatType\":\"default\"},\"from\":\"cols\"},{\"name\":\"name\",\"visualType\":\"string\",\"type\":\"category\",\"config\":true,\"field\":{\"alias\":\"\",\"desc\":\"\",\"useExpression\":false},\"format\":{\"formatType\":\"default\"},\"from\":\"cols\"}],\"rows\":[],\"metrics\":[{\"name\":\"score@Visualis@6F01974E\",\"visualType\":\"number\",\"type\":\"value\",\"agg\":\"sum\",\"config\":true,\"chart\":{\"id\":1,\"name\":\"table\",\"title\":\"表格\",\"icon\":\"icon-table\",\"coordinate\":\"other\",\"rules\":[{\"dimension\":[0,9999],\"metric\":[0,9999]}],\"data\":{\"cols\":{\"title\":\"列\",\"type\":\"category\"},\"rows\":{\"title\":\"行\",\"type\":\"category\"},\"metrics\":{\"title\":\"指标\",\"type\":\"value\"},\"filters\":{\"title\":\"筛选\",\"type\":\"all\"}},\"style\":{\"table\":{\"fontFamily\":\"PingFang SC\",\"fontSize\":\"12\",\"color\":\"#666\",\"lineStyle\":\"solid\",\"lineColor\":\"#D9D9D9\",\"headerBackgroundColor\":\"#f7f7f7\",\"headerConfig\":[],\"columnsConfig\":[],\"leftFixedColumns\":[],\"rightFixedColumns\":[],\"headerFixed\":true,\"autoMergeCell\":false,\"bordered\":true,\"size\":\"default\",\"withPaging\":true,\"pageSize\":\"20\",\"withNoAggregators\":false},\"spec\":{}}},\"field\":{\"alias\":\"\",\"desc\":\"\",\"useExpression\":false},\"format\":{\"formatType\":\"default\"},\"from\":\"metrics\"}],\"filters\":[],\"color\":{\"title\":\"颜色\",\"type\":\"category\",\"value\":{\"all\":\"#509af2\"},\"items\":[]},\"chartStyles\":{\"richText\":{\"content\":\"<p>〖@dv_name_dv@〗</p><p><strong style=\\\"color: rgb(230, 0, 0);\\\">〖@dv_birthday_dv@〗</strong></p>\"},\"spec\":{}},\"selectedChart\":15,\"data\":[],\"pagination\":{\"pageNo\":0,\"pageSize\":0,\"withPaging\":false,\"totalCount\":0},\"dimetionAxis\":\"col\",\"renderType\":\"rerender\",\"orders\":[],\"mode\":\"chart\",\"model\":{\"birthday\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"score\":{\"sqlType\":\"DOUBLE\",\"visualType\":\"number\",\"modelType\":\"value\"},\"teacher\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"city\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"sex\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"fee\":{\"sqlType\":\"DOUBLE\",\"visualType\":\"number\",\"modelType\":\"value\"},\"name\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"lesson\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"id\":{\"sqlType\":\"INT\",\"visualType\":\"number\",\"modelType\":\"value\"},\"class\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"exam_date\":{\"sqlType\":\"STRING\",\"visualType\":\"string\",\"modelType\":\"category\"},\"age\":{\"sqlType\":\"INT\",\"visualType\":\"number\",\"modelType\":\"value\"}},\"controls\":[],\"computed\":[],\"cache\":false,\"expired\":300,\"autoLoadData\":true,\"query\":{\"groups\":[\"birthday\",\"name\"],\"aggregators\":[{\"column\":\"score\",\"func\":\"sum\"}],\"filters\":[],\"orders\":[],\"pageNo\":0,\"pageSize\":0,\"nativeQuery\":false,\"cache\":false,\"expired\":0,\"flush\":false}}";
+ Set<String> columns = getWidgetUsedColumns(test);
+ columns.size();
+ }
+
+ private Set<String> getWidgetUsedColumns(String config){
+ Set<String> columns = Sets.newHashSet();
+ JsonObject configJson = BDPJettyServerHelper.gson().fromJson(config, JsonElement.class).getAsJsonObject();
+ configJson.getAsJsonArray("rows").forEach(e -> columns.add(getRealColumn(e.getAsJsonObject().get("name").getAsString())));
+ configJson.getAsJsonArray("cols").forEach(e -> columns.add(getRealColumn(e.getAsJsonObject().get("name").getAsString())));
+ configJson.getAsJsonArray("metrics").forEach(e -> columns.add(getRealColumn(e.getAsJsonObject().get("name").getAsString())));
+ return columns;
+ }
+
+ private String getRealColumn(String wrappedColumn){
+ return wrappedColumn.split("@")[0];
+ }
+}
diff --git a/contextservice/cs-server/bin/start-cs-server.sh b/contextservice/cs-server/bin/start-cs-server.sh
new file mode 100644
index 0000000..6278ff3
--- /dev/null
+++ b/contextservice/cs-server/bin/start-cs-server.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+export SERVER_LOG_PATH=$HOME/logs
+export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication
+
+if test -z "$SERVER_HEAP_SIZE"
+then
+ export SERVER_HEAP_SIZE="512M"
+fi
+
+if test -z "$SERVER_JAVA_OPTS"
+then
+ export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log"
+fi
+
+if [[ -f "${SERVER_PID}" ]]; then
+ pid=$(cat ${SERVER_PID})
+ if kill -0 ${pid} >/dev/null 2>&1; then
+ echo "Server is already running."
+ exit 1
+ fi
+fi
+
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+pid=$!
+if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME start failed!"
+ exit 1
+else
+ echo "server $SERVER_NAME start succeeded!"
+ echo $pid > $SERVER_PID
+ sleep 1
+fi
\ No newline at end of file
diff --git a/contextservice/cs-server/bin/stop--cs-server.sh b/contextservice/cs-server/bin/stop--cs-server.sh
new file mode 100644
index 0000000..9e84be4
--- /dev/null
+++ b/contextservice/cs-server/bin/stop--cs-server.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+
+function wait_for_server_to_die() {
+ local pid
+ local count
+ pid=$1
+ timeout=$2
+ count=0
+ timeoutTime=$(date "+%s")
+ let "timeoutTime+=$timeout"
+ currentTime=$(date "+%s")
+ forceKill=1
+
+ while [[ $currentTime -lt $timeoutTime ]]; do
+ $(kill ${pid} > /dev/null 2> /dev/null)
+ if kill -0 ${pid} > /dev/null 2>&1; then
+ sleep 3
+ else
+ forceKill=0
+ break
+ fi
+ currentTime=$(date "+%s")
+ done
+
+ if [[ forceKill -ne 0 ]]; then
+ $(kill -9 ${pid} > /dev/null 2> /dev/null)
+ fi
+}
+
+if [[ ! -f "${SERVER_PID}" ]]; then
+ echo "server $SERVER_NAME is not running"
+else
+ pid=$(cat ${SERVER_PID})
+ if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME is not running"
+ else
+ wait_for_server_to_die $pid 40
+ $(rm -f ${SERVER_PID})
+ echo "server $SERVER_NAME is stopped."
+ fi
+fi
\ No newline at end of file
diff --git a/contextservice/cs-server/conf/application.yml b/contextservice/cs-server/conf/application.yml
new file mode 100644
index 0000000..5d3e5e5
--- /dev/null
+++ b/contextservice/cs-server/conf/application.yml
@@ -0,0 +1,44 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+server:
+ port: 9042
+spring:
+ application:
+ name: cloud-contextservice
+
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: locahost
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
+
+pagehelper:
+ helper-dialect: mysql
+ reasonable: true
+ support-methods-arguments: true
+ params: countSql
\ No newline at end of file
diff --git a/contextservice/cs-server/conf/linkis.properties b/contextservice/cs-server/conf/linkis.properties
new file mode 100644
index 0000000..78e8806
--- /dev/null
+++ b/contextservice/cs-server/conf/linkis.properties
@@ -0,0 +1,22 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/dss_dev_center?characterEncoding=UTF-8
+wds.linkis.server.mybatis.datasource.username=
+wds.linkis.server.mybatis.datasource.password=
+wds.linkis.server.version=v1
+##restful
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.cs.server.restful
+##mybatis
+wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/linkis/cs/persistence/dao/impl/*.xml
+wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.linkis.cs.persistence.entity
+wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.linkis.cs.persistence.dao
diff --git a/contextservice/cs-server/conf/log4j.properties b/contextservice/cs-server/conf/log4j.properties
new file mode 100644
index 0000000..de6691a
--- /dev/null
+++ b/contextservice/cs-server/conf/log4j.properties
@@ -0,0 +1,26 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+log4j.rootCategory=INFO,console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern=%d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.logger.org.springframework=INFO
\ No newline at end of file
diff --git a/contextservice/cs-server/conf/log4j2.xml b/contextservice/cs-server/conf/log4j2.xml
new file mode 100644
index 0000000..1c68190
--- /dev/null
+++ b/contextservice/cs-server/conf/log4j2.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/contextservice/cs-server/pom.xml b/contextservice/cs-server/pom.xml
new file mode 100644
index 0000000..eb2c335
--- /dev/null
+++ b/contextservice/cs-server/pom.xml
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-server</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-cache</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-listener</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-persistence</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-highavailable</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-search</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cloudRPC</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-scheduler</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>add-source</goal>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>linkis-cs-server</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/java</directory>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>**/*.properties</exclude>
+ <exclude>**/application.yml</exclude>
+ <exclude>**/bootstrap.yml</exclude>
+ <exclude>**/log4j2.xml</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ </build>
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-server/src/main/assembly/distribution.xml b/contextservice/cs-server/src/main/assembly/distribution.xml
new file mode 100644
index 0000000..ac1d394
--- /dev/null
+++ b/contextservice/cs-server/src/main/assembly/distribution.xml
@@ -0,0 +1,71 @@
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-cs-server</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>linkis-cs-server</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>false</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/conf</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/bin</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>.</directory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <outputDirectory>logs</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/conf/ContextServerConf.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/conf/ContextServerConf.java
new file mode 100644
index 0000000..b614d0e
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/conf/ContextServerConf.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.conf;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 17:04
+ */
+public class ContextServerConf {
+
+ public final static String KEYWORD_SCAN_PACKAGE = CommonVars.apply("wds.linkis.cs.keyword.scan.package","com.webank.wedatasphere.linkis.cs").getValue();
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/enumeration/ServiceMethod.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/enumeration/ServiceMethod.java
new file mode 100644
index 0000000..fbc241a
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/enumeration/ServiceMethod.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.enumeration;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public enum ServiceMethod {
+ /**
+ *
+ */
+ CREATE, GET, SEARCH, REMOVE, REMOVEALL, UPDATE, RESET, SET, BIND, HEARTBEAT
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/enumeration/ServiceType.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/enumeration/ServiceType.java
new file mode 100644
index 0000000..a0432ed
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/enumeration/ServiceType.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.enumeration;
+
+import com.webank.wedatasphere.linkis.cs.server.protocol.*;
+
+/**
+ * Created by patinousward on 2020/2/19.
+ */
+public enum ServiceType {
+ /**
+ *
+ */
+ CONTEXT_ID {
+ @Override
+ public HttpRequestProtocol getRequestProtocol() {
+ return new ContextIDProtocol();
+ }
+ },
+ CONTEXT {
+ @Override
+ public HttpRequestProtocol getRequestProtocol() {
+ return new ContextProtocol();
+ }
+ },
+
+ CONTEXT_LISTENER {
+ @Override
+ public HttpRequestProtocol getRequestProtocol() {
+ return new ContextListenerProtocol();
+ }
+ };
+
+ public HttpRequestProtocol getRequestProtocol() {
+ return null;
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/DefaultKeywordParser.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/DefaultKeywordParser.java
new file mode 100644
index 0000000..2fb9a3e
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/DefaultKeywordParser.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.parser;
+
+import com.webank.wedatasphere.linkis.cs.common.annotation.KeywordMethod;
+import com.webank.wedatasphere.linkis.cs.server.conf.ContextServerConf;
+import org.apache.commons.lang.StringUtils;
+import org.reflections.ReflectionUtils;
+import org.reflections.Reflections;
+import org.reflections.scanners.MethodAnnotationsScanner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.lang.reflect.Method;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 16:19
+ */
+@Component
+public class DefaultKeywordParser implements KeywordParser {
+
+ private static final Logger logger = LoggerFactory.getLogger(DefaultKeywordParser.class);
+
+
+ Map<String, Set<KeywordMethodEntity>> keywordMethods = new HashMap<>();
+
+ Map<String, Set<Class<?>>> classRecord = new HashMap<>();
+
+
+ @PostConstruct
+ private void init() {
+ logger.info("init keyValueParser");
+ scanKeywordMethods();
+ }
+
+ private void scanKeywordMethods() {
+ Reflections reflections = new Reflections(ContextServerConf.KEYWORD_SCAN_PACKAGE, new MethodAnnotationsScanner());
+ Set<Method> methods = reflections.getMethodsAnnotatedWith(KeywordMethod.class);
+ Iterator<Method> iterator = methods.iterator();
+ while (iterator.hasNext()) {
+ Method method = iterator.next();
+ method.setAccessible(true);
+
+ KeywordMethod annotation = method.getAnnotation(KeywordMethod.class);
+ KeywordMethodEntity keywordMethodEntity = new KeywordMethodEntity();
+ keywordMethodEntity.setMethod(method);
+ keywordMethodEntity.setRegex(annotation.regex());
+ keywordMethodEntity.setSplitter(annotation.splitter());
+
+ String className = method.getDeclaringClass().getName();
+ if (!keywordMethods.containsKey(className)) {
+ keywordMethods.put(className, new HashSet<>());
+ }
+ keywordMethods.get(className).add(keywordMethodEntity);
+ }
+ }
+
+ /**
+ *
+ * @param obj
+ * @return
+ * @throws Exception
+ */
+ private Set<String> parseKeywords(Object obj) throws Exception {
+ Objects.requireNonNull(obj);
+ Set<String> keywords = new HashSet<>();
+ String className = obj.getClass().getName();
+ if (!classRecord.containsKey(className)) {
+ classRecord.put(className, ReflectionUtils.getAllSuperTypes(obj.getClass()));
+ }
+
+ Iterator<Class<?>> classIterator = classRecord.get(className).iterator();
+ while (classIterator.hasNext()) {
+ Class<?> clazz = classIterator.next();
+ if (! keywordMethods.containsKey(clazz.getName())) {
+ continue;
+ }
+ Iterator<KeywordMethodEntity> keywordMethodEntityIterator = keywordMethods.get(clazz.getName()).iterator();
+ while (keywordMethodEntityIterator.hasNext()) {
+ KeywordMethodEntity methodEntity = keywordMethodEntityIterator.next();
+ Object methodReturn = methodEntity.getMethod().invoke(obj);
+ if (null == methodReturn || StringUtils.isBlank(methodReturn.toString())) {
+ continue;
+ }
+ if (StringUtils.isNotBlank(methodEntity.getSplitter())) {
+ Collections.addAll(keywords, methodReturn.toString().split(methodEntity.getSplitter()));
+ } else if (StringUtils.isNotBlank(methodEntity.getRegex())) {
+ keywords.addAll(getString(methodReturn.toString(), methodEntity.getRegex()));
+ } else {
+ keywords.add(methodReturn.toString());
+ }
+ }
+ }
+ return keywords;
+ }
+
+ @Override
+ public Set<String> parse(Object obj) {
+ //先解析key
+ Set<String> keywords = new HashSet<>();
+ try {
+ keywords = parseKeywords(obj);
+ } catch (Exception e){
+ logger.error("Failed to parse keywords ", e);
+ }
+ return keywords;
+ }
+
+ private Set<String> getString(String s, String regex) {
+
+ Set<String> keywords = new HashSet<>();
+ Pattern p = Pattern.compile(regex);
+ Matcher m = p.matcher(s);
+ while(m.find()) {
+ keywords.add(m.group());
+
+ }
+ return keywords;
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/KeywordMethodEntity.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/KeywordMethodEntity.java
new file mode 100644
index 0000000..b71e491
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/KeywordMethodEntity.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.parser;
+
+import java.lang.reflect.Method;
+
+/**
+ * @author peacewong
+ * @date 2020/2/9 16:41
+ */
+public class KeywordMethodEntity {
+
+ private Method method;
+
+ private String splitter;
+
+ private String regex;
+
+ public Method getMethod() {
+ return method;
+ }
+
+ public void setMethod(Method method) {
+ this.method = method;
+ }
+
+ public String getSplitter() {
+ return splitter;
+ }
+
+ public void setSplitter(String splitter) {
+ this.splitter = splitter;
+ }
+
+ public String getRegex() {
+ return regex;
+ }
+
+ public void setRegex(String regex) {
+ this.regex = regex;
+ }
+
+ @Override
+ public int hashCode() {
+ return method.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return method.equals(obj);
+ }
+
+ @Override
+ public String toString() {
+ return super.toString();
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/KeywordParser.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/KeywordParser.java
new file mode 100644
index 0000000..a719ce1
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/parser/KeywordParser.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.parser;
+
+import java.util.Set;
+
+/**
+ * @author peacewong
+ * @date 2020/2/18 19:01
+ */
+public interface KeywordParser {
+ Set<String> parse(Object obj);
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/AbstractHttpRequestProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/AbstractHttpRequestProtocol.java
new file mode 100644
index 0000000..0ee18bd
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/AbstractHttpRequestProtocol.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceMethod;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public abstract class AbstractHttpRequestProtocol implements HttpRequestProtocol {
+
+ private Object[] requestObjects;
+
+ private String username;
+
+ private ServiceMethod serviceMethod;
+
+ @Override
+ public Object[] getRequestObjects() {
+ return requestObjects;
+ }
+
+ @Override
+ public void setRequestObjects(Object[] requestObjects) {
+ this.requestObjects = requestObjects;
+ }
+
+ @Override
+ public String getUsername() {
+ return username;
+ }
+
+ @Override
+ public ServiceMethod getServiceMethod() {
+ return serviceMethod;
+ }
+
+ @Override
+ public void setServiceMethod(ServiceMethod serviceMethod) {
+ this.serviceMethod = serviceMethod;
+ }
+
+ @Override
+ public void setUsername(String username) {
+ this.username = username;
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextIDProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextIDProtocol.java
new file mode 100644
index 0000000..8daedb9
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextIDProtocol.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public class ContextIDProtocol extends AbstractHttpRequestProtocol {
+ @Override
+ public String getServiceName() {
+ return ServiceType.CONTEXT_ID.name();
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextListenerProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextListenerProtocol.java
new file mode 100644
index 0000000..277419b
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextListenerProtocol.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public class ContextListenerProtocol extends AbstractHttpRequestProtocol {
+
+ @Override
+ public String getServiceName() {
+ return ServiceType.CONTEXT_LISTENER.name();
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextProtocol.java
new file mode 100644
index 0000000..46027e8
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/ContextProtocol.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public class ContextProtocol extends AbstractHttpRequestProtocol {
+
+ @Override
+ public String getServiceName() {
+ return ServiceType.CONTEXT.name();
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpProtocol.java
new file mode 100644
index 0000000..8b415cd
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpProtocol.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+import com.webank.wedatasphere.linkis.protocol.Protocol;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public interface HttpProtocol extends Protocol {
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpRequestProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpRequestProtocol.java
new file mode 100644
index 0000000..f5d59fd
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpRequestProtocol.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceMethod;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public interface HttpRequestProtocol extends HttpProtocol {
+
+ Object[] getRequestObjects();
+
+ void setRequestObjects(Object[] objs);
+
+ String getServiceName();
+
+ void setUsername(String username);
+
+ String getUsername();
+
+ ServiceMethod getServiceMethod();
+
+ void setServiceMethod(ServiceMethod method);
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpResponseProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpResponseProtocol.java
new file mode 100644
index 0000000..c7cf3f6
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/HttpResponseProtocol.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public interface HttpResponseProtocol<T> extends HttpProtocol {
+
+ void waitForComplete() throws InterruptedException;
+
+ void waitTimeEnd(long mills) throws InterruptedException;
+
+ void notifyJob();
+
+ T get();
+
+ void set(T t);
+
+ Object getResponseData();
+
+ void setResponseData(Object responseData);
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/RestResponseProtocol.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/RestResponseProtocol.java
new file mode 100644
index 0000000..a36907e
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/protocol/RestResponseProtocol.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.protocol;
+
+import com.webank.wedatasphere.linkis.server.Message;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.StringUtils;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public class RestResponseProtocol implements HttpResponseProtocol<Message> {
+
+ private Logger logger = LoggerFactory.getLogger(getClass());
+
+ private final Object lock = new Object();
+
+ private Message message;
+
+ private Object responseData;
+
+ @Override
+ public void waitForComplete() throws InterruptedException {
+ synchronized (lock) {
+ lock.wait();
+ }
+ }
+
+ @Override
+ public void waitTimeEnd(long mills) throws InterruptedException {
+ logger.info(String.format("start to wait %smills until job complete", mills));
+ synchronized (lock) {
+ lock.wait(mills);
+ }
+ }
+
+ @Override
+ public void notifyJob() {
+ logger.info("notify the job");
+ synchronized (lock) {
+ lock.notify();
+ }
+ }
+
+ @Override
+ public Message get() {
+ return this.message;
+ }
+
+ @Override
+ public void set(Message message) {
+ this.message = message;
+ }
+
+ @Override
+ public Object getResponseData() {
+ return this.responseData;
+ }
+
+ @Override
+ public void setResponseData(Object responseData) {
+ this.responseData = responseData;
+ }
+
+ public void ok(String msg) {
+ if (message == null) {
+ message = new Message();
+ }
+ if (StringUtils.isEmpty(msg)) {
+ message.setMessage("OK");
+ } else {
+ message.setMessage(msg);
+ }
+ }
+
+ public void error(String msg, Throwable t) {
+ if (message == null) {
+ message = new Message();
+ message.setStatus(1);
+ }
+ message.setMessage(msg);
+ if (t != null) {
+ message.$less$less("stack", ExceptionUtils.getFullStackTrace(t));
+ }
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextIDRestfulApi.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextIDRestfulApi.java
new file mode 100644
index 0000000..2b33bcd
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextIDRestfulApi.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.restful;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant;
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceMethod;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.CsScheduler;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpAnswerJob;
+import com.webank.wedatasphere.linkis.server.Message;
+import org.codehaus.jackson.JsonNode;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+@Component
+@Path("/contextservice")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class ContextIDRestfulApi implements CsRestfulParent {
+
+ @Autowired
+ private CsScheduler csScheduler;
+
+ @POST
+ @Path("createContextID")
+ public Response createContextID(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, ClassNotFoundException, IOException, CSErrorException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.CREATE, contextID);
+ return Message.messageToResponse(generateResponse(answerJob, "contextId"));
+ }
+
+ @GET
+ @Path("getContextID")
+ public Response getContextID(@Context HttpServletRequest req, @QueryParam("contextId") String id) throws InterruptedException, CSErrorException {
+ if (StringUtils.isEmpty(id)) {
+ throw new CSErrorException(97000, "contxtId cannot be empty");
+ }
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.GET, id);
+ Message message = generateResponse(answerJob, "contextID");
+ return Message.messageToResponse(message);
+ }
+
+ @POST
+ @Path("updateContextID")
+ public Response updateContextID(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ if (StringUtils.isEmpty(contextID.getContextId())) {
+ throw new CSErrorException(97000, "contxtId cannot be empty");
+ }
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.UPDATE, contextID);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("resetContextID")
+ public Response resetContextID(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException {
+ if (!jsonNode.has(ContextHTTPConstant.CONTEXT_ID_STR)) {
+ throw new CSErrorException(97000, ContextHTTPConstant.CONTEXT_ID_STR + " cannot be empty");
+ }
+ String id = jsonNode.get(ContextHTTPConstant.CONTEXT_ID_STR).getTextValue();
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.RESET, id);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+
+ @POST
+ @Path("removeContextID")
+ public Response removeContextID(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException {
+ String id = jsonNode.get("contextId").getTextValue();
+ if (StringUtils.isEmpty(id)) {
+ throw new CSErrorException(97000, "contxtId cannot be empty");
+ }
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.REMOVE, id);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @Override
+ public ServiceType getServiceType() {
+ return ServiceType.CONTEXT_ID;
+ }
+
+ @Override
+ public CsScheduler getScheduler() {
+ return this.csScheduler;
+ }
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextListenerRestfulApi.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextListenerRestfulApi.java
new file mode 100644
index 0000000..5b9fa70
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextListenerRestfulApi.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.restful;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceMethod;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.CsScheduler;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpAnswerJob;
+import com.webank.wedatasphere.linkis.server.Message;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+@Component
+@Path("/contextservice")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class ContextListenerRestfulApi implements CsRestfulParent {
+
+ @Autowired
+ private CsScheduler csScheduler;
+
+ private ObjectMapper objectMapper = new ObjectMapper();
+
+ @POST
+ @Path("onBindIDListener")
+ public Response onBindIDListener(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ String source = jsonNode.get("source").getTextValue();
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ ContextIDListenerDomain listener = new CommonContextIDListenerDomain();
+ listener.setSource(source);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.BIND, contextID, listener);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("onBindKeyListener")
+ public Response onBindKeyListener(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ String source = jsonNode.get("source").getTextValue();
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ ContextKey contextKey = getContextKeyFromJsonNode(jsonNode);
+ CommonContextKeyListenerDomain listener = new CommonContextKeyListenerDomain();
+ listener.setSource(source);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.BIND, contextID, contextKey, listener);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("heartbeat")
+ public Response heartbeat(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, IOException, CSErrorException {
+ String source = jsonNode.get("source").getTextValue();
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.HEARTBEAT, source);
+ return Message.messageToResponse(generateResponse(answerJob, "ContextKeyValueBean"));
+ }
+
+ @Override
+ public ServiceType getServiceType() {
+ return ServiceType.CONTEXT_LISTENER;
+ }
+
+ @Override
+ public CsScheduler getScheduler() {
+ return this.csScheduler;
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextRestfulApi.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextRestfulApi.java
new file mode 100644
index 0000000..ea2e1cf
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/ContextRestfulApi.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.restful;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceMethod;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.CsScheduler;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpAnswerJob;
+import com.webank.wedatasphere.linkis.server.Message;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.Map;
+
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+@Component
+@Path("contextservice")
+@Produces(MediaType.APPLICATION_JSON)
+@Consumes(MediaType.APPLICATION_JSON)
+public class ContextRestfulApi implements CsRestfulParent {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ContextRestfulApi.class);
+
+ @Autowired
+ private CsScheduler csScheduler;
+
+ private ObjectMapper objectMapper = new ObjectMapper();
+
+ @POST
+ @Path("getContextValue")
+ public Response getContextValue(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ ContextKey contextKey = getContextKeyFromJsonNode(jsonNode);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.GET, contextID, contextKey);
+ Message message = generateResponse(answerJob, "contextValue");
+ return Message.messageToResponse(message);
+ }
+
+
+ @POST
+ @Path("searchContextValue")
+ public Response searchContextValue(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ JsonNode condition = jsonNode.get("condition");
+ Map<Object, Object> conditionMap = objectMapper.convertValue(condition, new TypeReference<Map<Object, Object>>() {
+ });
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.SEARCH, contextID, conditionMap);
+ Message message = generateResponse(answerJob, "contextKeyValue");
+ return Message.messageToResponse(message);
+ }
+
+/* @GET
+ @Path("searchContextValueByCondition")
+ public Response searchContextValueByCondition(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException {
+ Condition condition = null;
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.SEARCH, condition);
+ return generateResponse(answerJob,"");
+ }*/
+
+
+ @POST
+ @Path("setValueByKey")
+ public Response setValueByKey(@Context HttpServletRequest req, JsonNode jsonNode) throws CSErrorException, IOException, ClassNotFoundException, InterruptedException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ ContextKey contextKey = getContextKeyFromJsonNode(jsonNode);
+ ContextValue contextValue = getContextValueFromJsonNode(jsonNode);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.SET, contextID, contextKey, contextValue);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("setValue")
+ public Response setValue(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ ContextKeyValue contextKeyValue = getContextKeyValueFromJsonNode(jsonNode);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.SET, contextID, contextKeyValue);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("resetValue")
+ public Response resetValue(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ ContextKey contextKey = getContextKeyFromJsonNode(jsonNode);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.RESET, contextID, contextKey);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("removeValue")
+ public Response removeValue(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ ContextKey contextKey = getContextKeyFromJsonNode(jsonNode);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.REMOVE, contextID, contextKey);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("removeAllValue")
+ public Response removeAllValue(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.REMOVEALL, contextID);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("removeAllValueByKeyPrefixAndContextType")
+ public Response removeAllValueByKeyPrefixAndContextType(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ String contextType = jsonNode.get(ContextHTTPConstant.CONTEXT_KEY_TYPE_STR).getTextValue();
+ String keyPrefix = jsonNode.get(ContextHTTPConstant.CONTEXT_KEY_PREFIX_STR).getTextValue();
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.REMOVEALL, contextID, ContextType.valueOf(contextType),keyPrefix);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @POST
+ @Path("removeAllValueByKeyPrefix")
+ public Response removeAllValueByKeyPrefix(@Context HttpServletRequest req, JsonNode jsonNode) throws InterruptedException, CSErrorException, IOException, ClassNotFoundException {
+ ContextID contextID = getContextIDFromJsonNode(jsonNode);
+ String keyPrefix = jsonNode.get(ContextHTTPConstant.CONTEXT_KEY_PREFIX_STR).getTextValue();
+ HttpAnswerJob answerJob = submitRestJob(req, ServiceMethod.REMOVEALL, contextID,keyPrefix);
+ return Message.messageToResponse(generateResponse(answerJob, ""));
+ }
+
+ @Override
+ public ServiceType getServiceType() {
+ return ServiceType.CONTEXT;
+ }
+
+ @Override
+ public CsScheduler getScheduler() {
+ return this.csScheduler;
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/CsRestfulParent.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/CsRestfulParent.java
new file mode 100644
index 0000000..f34c482
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/restful/CsRestfulParent.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.restful;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceMethod;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpRequestProtocol;
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpResponseProtocol;
+import com.webank.wedatasphere.linkis.cs.server.protocol.RestResponseProtocol;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.CsScheduler;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpAnswerJob;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.RestJobBuilder;
+import com.webank.wedatasphere.linkis.cs.server.util.CsUtils;
+import com.webank.wedatasphere.linkis.server.Message;
+import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
+import org.codehaus.jackson.JsonNode;
+
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public interface CsRestfulParent {
+
+ default HttpAnswerJob submitRestJob(HttpServletRequest req,
+ ServiceMethod method,
+ Object... objects) throws InterruptedException {
+ // TODO: 2020/3/3 单例
+ HttpAnswerJob job = (HttpAnswerJob) new RestJobBuilder().build(getServiceType());
+ HttpRequestProtocol protocol = job.getRequestProtocol();
+ protocol.setUsername(SecurityFilter.getLoginUsername(req));
+ protocol.setServiceMethod(method);
+ protocol.setRequestObjects(objects);
+ getScheduler().sumbit(job);
+ return job;
+ }
+
+ default Message generateResponse(HttpAnswerJob job, String responseKey) throws CSErrorException {
+ HttpResponseProtocol responseProtocol = job.getResponseProtocol();
+ if (responseProtocol instanceof RestResponseProtocol) {
+ Message message = ((RestResponseProtocol) responseProtocol).get();
+ if (message == null) {
+ return Message.error("job execute timeout");
+ }
+ int status = ((RestResponseProtocol) responseProtocol).get().getStatus();
+ if (status == 1) {
+ //failed
+ return ((RestResponseProtocol) responseProtocol).get();
+ } else if (status == 0) {
+ Object data = job.getResponseProtocol().getResponseData();
+ if (data == null) {
+ return Message.ok().data(responseKey, null);
+ } else if (data instanceof List && ((List) data).isEmpty()) {
+ return Message.ok().data(responseKey, new String[]{});
+ } else if (data instanceof List) {
+ ArrayList<String> strings = new ArrayList<>();
+ for (Object d : (List) data) {
+ strings.add(CsUtils.serialize(d));
+ }
+ return Message.ok().data(responseKey, strings);
+ } else {
+ String dataStr = CsUtils.serialize(data);
+ return Message.ok().data(responseKey, dataStr);
+ }
+ } else {
+
+ }
+ }
+ return Message.ok();
+ }
+
+ ServiceType getServiceType();
+
+ CsScheduler getScheduler();
+
+ default ContextID getContextIDFromJsonNode(JsonNode jsonNode) throws CSErrorException, IOException, ClassNotFoundException {
+ return deserialize(jsonNode, "contextID");
+ }
+
+ default <T> T deserialize(JsonNode jsonNode, String key) throws CSErrorException {
+ String str = jsonNode.get(key).getTextValue();
+ return (T) CsUtils.SERIALIZE.deserialize(str);
+ }
+
+ default ContextKey getContextKeyFromJsonNode(JsonNode jsonNode) throws CSErrorException, IOException, ClassNotFoundException {
+ return deserialize(jsonNode, "contextKey");
+ }
+
+ default ContextValue getContextValueFromJsonNode(JsonNode jsonNode) throws CSErrorException, IOException, ClassNotFoundException {
+ return deserialize(jsonNode, "contextValue");
+ }
+
+ default ContextKeyValue getContextKeyValueFromJsonNode(JsonNode jsonNode) throws CSErrorException, IOException, ClassNotFoundException {
+ return deserialize(jsonNode, "contextKeyValue");
+ }
+
+
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/CsScheduler.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/CsScheduler.java
new file mode 100644
index 0000000..7cbcb99
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/CsScheduler.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+import com.webank.wedatasphere.linkis.cs.server.service.Service;
+
+/**
+ * Created by patinousward on 2020/2/21.
+ */
+public interface CsScheduler {
+
+ void addService(Service service);
+
+ Service[] getServices();
+
+ void sumbit(HttpJob job) throws InterruptedException;
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/DefaultCsScheduler.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/DefaultCsScheduler.java
new file mode 100644
index 0000000..1070444
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/DefaultCsScheduler.java
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+import com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl.CsJobListener;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl.CsSchedulerJob;
+import com.webank.wedatasphere.linkis.cs.server.service.Service;
+import com.webank.wedatasphere.linkis.scheduler.Scheduler;
+import com.webank.wedatasphere.linkis.scheduler.queue.Group;
+import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory;
+import com.webank.wedatasphere.linkis.scheduler.queue.Job;
+import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelGroup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Created by patinousward on 2020/2/21.
+ */
+@Component
+public class DefaultCsScheduler implements CsScheduler {
+
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ @Autowired
+ private Scheduler scheduler;
+
+ @Autowired
+ private List<Service> services;
+
+ @Override
+ public void addService(Service service) {
+ services.add(service);
+ }
+
+ @Override
+ public Service[] getServices() {
+ return this.services.toArray(new Service[]{});
+ }
+
+ @Override
+ public void sumbit(HttpJob job) throws InterruptedException {
+ // TODO: 2020/3/3 参数配置化
+ GroupFactory groupFactory = scheduler.getSchedulerContext().getOrCreateGroupFactory();
+ Group group = groupFactory.getOrCreateGroup(job.getRequestProtocol().getUsername());
+ if (group instanceof ParallelGroup) {
+ ParallelGroup parallelGroup = (ParallelGroup) group;
+ if (parallelGroup.getMaxRunningJobs() == 0) {
+ parallelGroup.setMaxRunningJobs(100);
+ }
+ if (parallelGroup.getMaxAskExecutorTimes() == 0) {
+ parallelGroup.setMaxAskExecutorTimes(1000);
+ }
+ }
+ //create csJob
+ Job csJob = buildJob(job);
+ //注册listener
+ csJob.setJobListener(new CsJobListener());
+ scheduler.submit(csJob);
+ if (job instanceof HttpAnswerJob) {
+ HttpAnswerJob answerJob = (HttpAnswerJob) job;
+ answerJob.getResponseProtocol().waitTimeEnd(5000);
+ }
+ }
+
+ private Job buildJob(HttpJob job) {
+ CsSchedulerJob csJob = new CsSchedulerJob();
+ //暂时将groupName给jobid
+ csJob.setId(job.getRequestProtocol().getUsername());
+ csJob.set(job);
+ //从多个serveice中找出一个合适执行的service
+ Optional<Service> service = Arrays.stream(getServices()).filter(s -> s.ifAccept(job)).findFirst();
+ if (service.isPresent()) {
+ logger.info(String.format("find %s service to execute job",service.get().getName()));
+ csJob.setConsuemr(service.get()::accept);
+ }
+ return csJob;
+ }
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpAnswerJob.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpAnswerJob.java
new file mode 100644
index 0000000..a6e64ab
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpAnswerJob.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpResponseProtocol;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public interface HttpAnswerJob extends HttpJob {
+
+ HttpResponseProtocol getResponseProtocol();
+
+ void setResponseProtocol(HttpResponseProtocol protocol);
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpAnswerJobBuilder.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpAnswerJobBuilder.java
new file mode 100644
index 0000000..ff56931
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpAnswerJobBuilder.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpRequestProtocol;
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpResponseProtocol;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public abstract class HttpAnswerJobBuilder extends HttpJobBuilder {
+
+ @Override
+ public final HttpJob build(ServiceType serviceType) {
+ return buildResponseProtocol(super.build(serviceType));
+ }
+
+ @Override
+ protected final HttpJob createHttpJob() {
+ return new DefaultHttpAnswerJob();
+ }
+
+ protected abstract HttpJob buildResponseProtocol(HttpJob job);
+
+ private static class DefaultHttpAnswerJob implements HttpAnswerJob {
+ private HttpRequestProtocol httpRequestProtocol;
+
+ private HttpResponseProtocol httpResponseProtocol;
+
+ @Override
+ public HttpRequestProtocol getRequestProtocol() {
+ return this.httpRequestProtocol;
+ }
+
+ @Override
+ public void setRequestProtocol(HttpRequestProtocol protocol) {
+ this.httpRequestProtocol = protocol;
+ }
+
+ @Override
+ public HttpResponseProtocol getResponseProtocol() {
+ return this.httpResponseProtocol;
+ }
+
+ @Override
+ public void setResponseProtocol(HttpResponseProtocol protocol) {
+ this.httpResponseProtocol = protocol;
+ }
+ }
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpJob.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpJob.java
new file mode 100644
index 0000000..4d7fe00
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpJob.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpRequestProtocol;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public interface HttpJob {
+
+ HttpRequestProtocol getRequestProtocol();
+
+ void setRequestProtocol(HttpRequestProtocol protocol);
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpJobBuilder.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpJobBuilder.java
new file mode 100644
index 0000000..51e95d4
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpJobBuilder.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public abstract class HttpJobBuilder {
+
+ public HttpJob build(ServiceType serviceType) {
+ return buildRequestProtocol(createHttpJob(), serviceType);
+ }
+
+ private HttpJob buildRequestProtocol(HttpJob job, ServiceType serviceType) {
+ job.setRequestProtocol(serviceType.getRequestProtocol());
+ return job;
+ }
+
+ protected abstract HttpJob createHttpJob();
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpPriorityJob.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpPriorityJob.java
new file mode 100644
index 0000000..7f1602c
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/HttpPriorityJob.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public interface HttpPriorityJob extends HttpJob {
+
+ int getPriority();
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/RestJobBuilder.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/RestJobBuilder.java
new file mode 100644
index 0000000..37df82d
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/RestJobBuilder.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler;
+
+import com.webank.wedatasphere.linkis.cs.server.protocol.RestResponseProtocol;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public class RestJobBuilder extends HttpAnswerJobBuilder {
+
+ @Override
+ protected final HttpJob buildResponseProtocol(HttpJob job) {
+ RestResponseProtocol protocol = new RestResponseProtocol();
+ ((HttpAnswerJob) job).setResponseProtocol(protocol);
+ return job;
+ }
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecuteRequest.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecuteRequest.java
new file mode 100644
index 0000000..3a85cda
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecuteRequest.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl;
+
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpJob;
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest;
+
+import java.util.function.Consumer;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public class CsExecuteRequest implements ExecuteRequest, JobToExecuteRequestConsumer<HttpJob> {
+
+ private HttpJob httpJob;
+
+ // TODO: 2020/3/3 变量名修改
+ private Consumer<HttpJob> jobConsumer;
+
+ @Override
+ public String code() {
+ return null;
+ }
+
+
+ @Override
+ public HttpJob get() {
+ return this.httpJob;
+ }
+
+ @Override
+ public void set(HttpJob httpJob) {
+ this.httpJob = httpJob;
+ }
+
+ @Override
+ public Consumer<HttpJob> getConsumer() {
+ return this.jobConsumer;
+ }
+
+ @Override
+ public void setConsuemr(Consumer<HttpJob> jobConsumer) {
+ this.jobConsumer = jobConsumer;
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecutor.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecutor.java
new file mode 100644
index 0000000..965c64b
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecutor.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl;
+
+import com.webank.wedatasphere.linkis.protocol.engine.EngineState$;
+import com.webank.wedatasphere.linkis.scheduler.executer.*;
+import scala.Enumeration;
+
+import java.io.IOException;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public class CsExecutor implements Executor {
+
+ private long id;
+ private Enumeration.Value state;
+
+ @Override
+ public long getId() {
+ return this.id;
+ }
+
+ @Override
+ public ExecuteResponse execute(ExecuteRequest executeRequest) {
+ //httpjob执行的地方
+ try {
+ if (executeRequest instanceof CsExecuteRequest) {
+ CsExecuteRequest request = (CsExecuteRequest) executeRequest;
+ request.getConsumer().accept(request.get());
+ }
+ return new SuccessExecuteResponse();
+ } catch (Exception e) {
+ return new ErrorExecuteResponse(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public EngineState$.Value state() {
+ return this.state;
+ }
+
+ @Override
+ public ExecutorInfo getExecutorInfo() {
+ return new ExecutorInfo(id, state);
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecutorManager.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecutorManager.java
new file mode 100644
index 0000000..08e55c9
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsExecutorManager.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl;
+
+import com.webank.wedatasphere.linkis.scheduler.executer.Executor;
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorManager;
+import com.webank.wedatasphere.linkis.scheduler.listener.ExecutorListener;
+import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEvent;
+import scala.Option;
+import scala.Some;
+import scala.concurrent.duration.Duration;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public class CsExecutorManager extends ExecutorManager {
+
+ @Override
+ public void setExecutorListener(ExecutorListener executorListener) {
+
+ }
+
+ @Override
+ public Executor createExecutor(SchedulerEvent event) {
+ return new CsExecutor();
+ }
+
+ @Override
+ public Option<Executor> askExecutor(SchedulerEvent event) {
+ return new Some<>(createExecutor(event));
+ }
+
+ @Override
+ public Option<Executor> askExecutor(SchedulerEvent event, Duration wait) {
+ return askExecutor(event);
+ }
+
+ @Override
+ public Option<Executor> getById(long id) {
+ return new Some<>(null);
+ }
+
+ @Override
+ public Executor[] getByGroup(String groupName) {
+ return new Executor[0];
+ }
+
+ @Override
+ public void delete(Executor executor) {
+
+ }
+
+ @Override
+ public void shutdown() {
+
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsJobListener.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsJobListener.java
new file mode 100644
index 0000000..9fb8d52
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsJobListener.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl;
+
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpResponseProtocol;
+import com.webank.wedatasphere.linkis.cs.server.protocol.RestResponseProtocol;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpAnswerJob;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpJob;
+import com.webank.wedatasphere.linkis.scheduler.executer.ErrorExecuteResponse;
+import com.webank.wedatasphere.linkis.scheduler.listener.JobListener;
+import com.webank.wedatasphere.linkis.scheduler.queue.Job;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public class CsJobListener implements JobListener {
+ @Override
+ public void onJobScheduled(Job job) {
+ //nothing to do
+ }
+
+ @Override
+ public void onJobInited(Job job) {
+ //nothing to do
+ }
+
+ @Override
+ public void onJobWaitForRetry(Job job) {
+ //nothing to do
+ }
+
+ @Override
+ public void onJobRunning(Job job) {
+ //nothing to do
+ }
+
+ @Override
+ public void onJobCompleted(Job job) {
+ // TODO: 2020/2/22 正常 /异常
+ if (job instanceof CsSchedulerJob) {
+ HttpJob httpJob = ((CsSchedulerJob) job).get();
+ if (httpJob instanceof HttpAnswerJob) {
+ HttpAnswerJob answerJob = (HttpAnswerJob) httpJob;
+ HttpResponseProtocol responseProtocol = answerJob.getResponseProtocol();
+ if (!job.isSucceed() && responseProtocol instanceof RestResponseProtocol) {
+ ErrorExecuteResponse errorResponse = job.getErrorResponse();
+ ((RestResponseProtocol) responseProtocol).error(errorResponse.message(), errorResponse.t());
+ }
+ answerJob.getResponseProtocol().notifyJob();
+ }
+ }
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsSchedulerBean.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsSchedulerBean.java
new file mode 100644
index 0000000..dc93657
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsSchedulerBean.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl;
+
+import com.webank.wedatasphere.linkis.scheduler.Scheduler;
+import com.webank.wedatasphere.linkis.scheduler.SchedulerContext;
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorManager;
+import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelScheduler;
+import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelSchedulerContextImpl;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+@Configuration
+@Import(CsExecutorManager.class)
+public class CsSchedulerBean {
+
+ @Bean
+ public SchedulerContext getSchedulerContext(CsExecutorManager csExecutorManager) {
+ return new ParallelSchedulerContextImpl(3000) {
+ @Override
+ public ExecutorManager getOrCreateExecutorManager() {
+ return csExecutorManager;
+ }
+ };
+ }
+
+ @Bean
+ public Scheduler getScheduler(SchedulerContext context) {
+ ParallelScheduler parallelScheduler = new ParallelScheduler(context);
+ parallelScheduler.init();
+ return parallelScheduler;
+ }
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsSchedulerJob.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsSchedulerJob.java
new file mode 100644
index 0000000..e7d0584
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/CsSchedulerJob.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl;
+
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpJob;
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest;
+import com.webank.wedatasphere.linkis.scheduler.queue.Job;
+import com.webank.wedatasphere.linkis.scheduler.queue.JobInfo;
+
+import java.io.IOException;
+import java.util.function.Consumer;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public class CsSchedulerJob extends Job implements JobToExecuteRequestConsumer<HttpJob> {
+
+ private HttpJob httpJob;
+
+ private Consumer<HttpJob> jobConsumer;
+
+ @Override
+ public HttpJob get() {
+ return this.httpJob;
+ }
+
+ @Override
+ public void set(HttpJob httpJob) {
+ this.httpJob = httpJob;
+ }
+
+ @Override
+ public Consumer<HttpJob> getConsumer() {
+ return this.jobConsumer;
+ }
+
+ @Override
+ public void setConsuemr(Consumer<HttpJob> jobConsumer) {
+ this.jobConsumer = jobConsumer;
+ }
+
+
+ @Override
+ public void init() {
+ // TODO: 2020/2/18
+ }
+
+ @Override
+ public ExecuteRequest jobToExecuteRequest() {
+ CsExecuteRequest request = new CsExecuteRequest();
+ request.set(httpJob);
+ request.setConsuemr(jobConsumer);
+ return request;
+ }
+
+ @Override
+ public String getName() {
+ return getId();
+ }
+
+ @Override
+ public JobInfo getJobInfo() {
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/JobToExecuteRequestConsumer.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/JobToExecuteRequestConsumer.java
new file mode 100644
index 0000000..3f93990
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/scheduler/linkisImpl/JobToExecuteRequestConsumer.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.scheduler.linkisImpl;
+
+import java.util.function.Consumer;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public interface JobToExecuteRequestConsumer<T> {
+
+ T get();
+
+ void set(T t);
+
+ Consumer<T> getConsumer();
+
+ void setConsuemr(Consumer<T> tConsumer);
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/AbstractService.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/AbstractService.java
new file mode 100644
index 0000000..e380a06
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/AbstractService.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSWarnException;
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpRequestProtocol;
+import com.webank.wedatasphere.linkis.cs.server.protocol.HttpResponseProtocol;
+import com.webank.wedatasphere.linkis.cs.server.protocol.RestResponseProtocol;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpAnswerJob;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Optional;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public abstract class AbstractService implements Service {
+
+ protected final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ @Override
+ public boolean ifAccept(HttpJob job) {
+ return getName().equals(job.getRequestProtocol().getServiceName());
+ }
+
+ @Override
+ public void accept(HttpJob job) throws CSWarnException {
+ try {
+ //根据参数类型和方法名,选出方法进行调用
+ HttpRequestProtocol protocol = job.getRequestProtocol();
+ Object[] params = protocol.getRequestObjects();
+ String method = protocol.getServiceMethod().name().toUpperCase();
+ Method[] methods = this.getClass().getMethods();
+ Optional<Method> first = Arrays.stream(methods).filter(f -> f.getName().toUpperCase().contains(method))
+ .filter(f -> f.getParameterTypes().length == params.length)
+ .filter(f -> judgeMethod(f, params)).findFirst();
+ Object response = first.orElseThrow(() -> new CSErrorException(97000, "can not find a method to invoke")).invoke(this, params);
+ if (job instanceof HttpAnswerJob) {
+ HttpResponseProtocol responseProtocol = ((HttpAnswerJob) job).getResponseProtocol();
+ if (responseProtocol instanceof RestResponseProtocol) {
+ ((RestResponseProtocol) responseProtocol).ok(null);
+ }
+ responseProtocol.setResponseData(response);
+ }
+ } catch (Exception e) {
+ logger.error(String.format("execute %s service failed:", getName()), e);
+ throw new CSWarnException(97000, e.getMessage());
+ }
+ }
+
+ private boolean judgeMethod(Method method, Object... objects) {
+ boolean flag = true;
+ //传入参数类型是否是方法参数的子类
+ Class<?>[] parameterTypes = method.getParameterTypes();
+ for (int i = 0; i < parameterTypes.length; i++) {
+ if (!parameterTypes[i].isAssignableFrom(objects[i].getClass())) {
+ flag = false;
+ break;
+ }
+ }
+ return flag;
+ }
+
+ @Override
+ public void init() {
+
+ }
+
+ @Override
+ public void start() {
+
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextIDService.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextIDService.java
new file mode 100644
index 0000000..ad8472a
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextIDService.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public abstract class ContextIDService extends AbstractService {
+
+ public abstract String createContextID(ContextID contextID) throws CSErrorException;
+
+ public abstract ContextID getContextID(String id) throws CSErrorException;
+
+ public abstract void updateContextID(ContextID contextID) throws CSErrorException;
+
+ public abstract void resetContextID(String id) throws CSErrorException;
+
+ public abstract void removeContextID(String id) throws CSErrorException;
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextListenerService.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextListenerService.java
new file mode 100644
index 0000000..66450b4
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextListenerService.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean;
+
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public abstract class ContextListenerService extends AbstractService {
+
+ public abstract void onBind(ContextID contextID, ContextIDListenerDomain listener) throws CSErrorException;
+
+ public abstract void onBind(ContextID contextID, ContextKey contextKey, ContextKeyListenerDomain listener) throws CSErrorException;
+
+ public abstract List<ContextKeyValueBean> heartbeat(String source);
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextService.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextService.java
new file mode 100644
index 0000000..64389be
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/ContextService.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.condition.Condition;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public abstract class ContextService extends AbstractService {
+
+ public abstract ContextValue getContextValue(ContextID contextID, ContextKey contextKey);
+
+ public abstract List<ContextKeyValue> searchContextValue(ContextID contextID, Map<Object, Object> conditionMap) throws ContextSearchFailedException;
+
+ //public abstract List<ContextKeyValue> searchContextValueByCondition(Condition condition) throws ContextSearchFailedException;
+
+ public abstract void setValueByKey(ContextID contextID, ContextKey contextKey, ContextValue contextValue) throws CSErrorException, ClassNotFoundException, JsonProcessingException;
+
+ public abstract void setValue(ContextID contextID, ContextKeyValue contextKeyValuee) throws CSErrorException, ClassNotFoundException, JsonProcessingException;
+
+ public abstract void resetValue(ContextID contextID, ContextKey contextKey) throws CSErrorException;
+
+ public abstract void removeValue(ContextID contextID, ContextKey contextKey) throws CSErrorException;
+
+ public abstract void removeAllValue(ContextID contextID) throws CSErrorException;
+
+ public abstract void removeAllValueByKeyPrefixAndContextType(ContextID contextID, ContextType contextType,String keyPrefix) throws CSErrorException;
+
+ public abstract void removeAllValueByKeyPrefix(ContextID contextID,String keyPrefix) throws CSErrorException;
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/Service.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/Service.java
new file mode 100644
index 0000000..fd28fd3
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/Service.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSWarnException;
+import com.webank.wedatasphere.linkis.cs.server.scheduler.HttpJob;
+
+import java.io.Closeable;
+
+/**
+ * Created by patinousward on 2020/2/18.
+ */
+public interface Service extends Closeable {
+
+ void init();
+
+ void start();
+
+ String getName();
+
+ boolean ifAccept(HttpJob job);
+
+ void accept(HttpJob job) throws CSWarnException;
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextIDServiceImpl.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextIDServiceImpl.java
new file mode 100644
index 0000000..96ba24f
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextIDServiceImpl.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.persistence.ContextPersistenceManager;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextIDPersistence;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.service.ContextIDService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * Created by patinousward on 2020/2/21.
+ */
+@Component
+public class ContextIDServiceImpl extends ContextIDService {
+
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ @Autowired
+ private ContextPersistenceManager persistenceManager;
+
+ private ContextIDPersistence getPersistence() throws CSErrorException {
+ return persistenceManager.getContextIDPersistence();
+ }
+
+ @Override
+ public String getName() {
+ return ServiceType.CONTEXT_ID.name();
+ }
+
+
+ @Override
+ public String createContextID(ContextID contextID) throws CSErrorException {
+ getPersistence().createContextID(contextID);
+ logger.info(String.format("createContextID,csId:%s", contextID.getContextId()));
+ return contextID.getContextId();
+ }
+
+ @Override
+ public ContextID getContextID(String id) throws CSErrorException {
+ logger.info(String.format("getContextID,csId:%s", id));
+ return getPersistence().getContextID(id);
+ }
+
+ @Override
+ public void updateContextID(ContextID contextID) throws CSErrorException {
+ logger.info(String.format("getContextID,csId:%s", contextID.getContextId()));
+ getPersistence().updateContextID(contextID);
+ }
+
+ @Override
+ public void resetContextID(String id) throws CSErrorException {
+ // TODO: 2020/2/23 reset 方法
+ }
+
+ @Override
+ public void removeContextID(String id) throws CSErrorException {
+ logger.info(String.format("removeContextID,csId:%s", id));
+ getPersistence().deleteContextID(id);
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextListenerServiceImpl.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextListenerServiceImpl.java
new file mode 100644
index 0000000..6fb7d73
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextListenerServiceImpl.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service.impl;
+
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.CommonContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextIDListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.listener.ContextKeyListenerDomain;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean;
+import com.webank.wedatasphere.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import com.webank.wedatasphere.linkis.cs.persistence.ContextPersistenceManager;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextIDListenerPersistence;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextKeyListenerPersistence;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.service.ContextListenerService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by patinousward on 2020/2/21.
+ */
+@Component
+public class ContextListenerServiceImpl extends ContextListenerService {
+
+ @Autowired
+ private ContextPersistenceManager persistenceManager;
+
+ private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+
+ private ContextIDListenerPersistence getIDListenerPersistence() throws CSErrorException {
+ return persistenceManager.getContextIDListenerPersistence();
+ }
+
+ private ContextKeyListenerPersistence getKeyListenerPersistence() throws CSErrorException {
+ return persistenceManager.getContextKeyListenerPersistence();
+ }
+
+ @Override
+ public String getName() {
+ return ServiceType.CONTEXT_LISTENER.name();
+ }
+
+
+ @Override
+ public void onBind(ContextID contextID, ContextIDListenerDomain domain) throws CSErrorException {
+ logger.info(String.format("onBind,csId:%s", contextID.getContextId()));
+ domain.setContextID(contextID);
+ getIDListenerPersistence().create(contextID, domain);
+ DefaultContextListenerManager instance = DefaultContextListenerManager.getInstance();
+ instance.getContextIDCallbackEngine().registerClient(domain);
+ }
+
+ @Override
+ public void onBind(ContextID contextID, ContextKey contextKey, ContextKeyListenerDomain domain) throws CSErrorException {
+ logger.info(String.format("onBind,csId:%s,key:%s", contextID.getContextId(), contextKey.getKey()));
+ domain.setContextKey(contextKey);
+ // TODO: 2020/2/28
+ if (domain instanceof CommonContextKeyListenerDomain) {
+ ((CommonContextKeyListenerDomain) domain).setContextID(contextID);
+ }
+ getKeyListenerPersistence().create(contextID, domain);
+ DefaultContextListenerManager instance = DefaultContextListenerManager.getInstance();
+ instance.getContextKeyCallbackEngine().registerClient(domain);
+ }
+
+ @Override
+ public List<ContextKeyValueBean> heartbeat(String clientSource) {
+ logger.info(String.format("heartbeat,clientSource:%s", clientSource));
+ DefaultContextListenerManager instance = DefaultContextListenerManager.getInstance();
+ ArrayList<ContextKeyValueBean> idCallback = instance.getContextIDCallbackEngine().getListenerCallback(clientSource);
+ ArrayList<ContextKeyValueBean> keyCallback = instance.getContextKeyCallbackEngine().getListenerCallback(clientSource);
+ idCallback.addAll(keyCallback);
+ return idCallback;
+ }
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextServiceImpl.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextServiceImpl.java
new file mode 100644
index 0000000..5511d14
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/service/impl/ContextServiceImpl.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.service.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.webank.wedatasphere.linkis.cs.ContextSearch;
+import com.webank.wedatasphere.linkis.cs.DefaultContextSearch;
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue;
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.contextcache.ContextCacheService;
+import com.webank.wedatasphere.linkis.cs.exception.ContextSearchFailedException;
+import com.webank.wedatasphere.linkis.cs.persistence.ContextPersistenceManager;
+import com.webank.wedatasphere.linkis.cs.persistence.entity.PersistenceContextKeyValue;
+import com.webank.wedatasphere.linkis.cs.persistence.persistence.ContextMapPersistence;
+import com.webank.wedatasphere.linkis.cs.server.enumeration.ServiceType;
+import com.webank.wedatasphere.linkis.cs.server.parser.KeywordParser;
+import com.webank.wedatasphere.linkis.cs.server.service.ContextService;
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Created by patinousward on 2020/2/21.
+ */
+@Component
+public class ContextServiceImpl extends ContextService {
+
+ //对接search
+
+ private ContextSearch contextSearch = new DefaultContextSearch();
+
+ @Autowired
+ private ContextCacheService contextCacheService;
+
+ @Autowired
+ private ContextPersistenceManager persistenceManager;
+
+ @Autowired
+ private KeywordParser keywordParser;
+
+
+ private ObjectMapper jackson = BDPJettyServerHelper.jacksonJson();
+
+
+ private ContextMapPersistence getPersistence() throws CSErrorException {
+ return persistenceManager.getContextMapPersistence();
+ }
+
+ @Override
+ public String getName() {
+ return ServiceType.CONTEXT.name();
+ }
+
+ @Override
+ public ContextValue getContextValue(ContextID contextID, ContextKey contextKey) {
+ //从缓存取即可,缓存中无会去数据库中拉取
+ ContextKeyValue keyValue = contextCacheService.get(contextID, contextKey);
+ if (keyValue == null) {
+ return null;
+ }
+ logger.info(String.format("getContextValue,csId:%s,key:%s,csType:%s,csScope:%s",
+ contextID.getContextId(), contextKey.getKey(), contextKey.getContextType(), contextKey.getContextScope()));
+ return keyValue.getContextValue();
+ }
+
+ @Override
+ public List<ContextKeyValue> searchContextValue(ContextID contextID, Map<Object, Object> conditionMap) throws ContextSearchFailedException {
+ //return List<ContextKeyValue>
+ logger.info(String.format("searchContextValue,csId:%s", contextID.getContextId()));
+ return contextSearch.search(contextCacheService, contextID, conditionMap);
+ }
+
+/* @Override
+ public List<ContextKeyValue> searchContextValueByCondition(Condition condition) throws ContextSearchFailedException {
+ //return List<ContextKeyValue>
+ return contextSearch.search(contextCacheService, null, condition);
+ }*/
+
+ @Override
+ public void setValueByKey(ContextID contextID, ContextKey contextKey, ContextValue contextValue) throws CSErrorException, ClassNotFoundException, JsonProcessingException {
+ //1.获取value
+ Object value = contextValue.getValue();
+ //2.解析keywords,放入contextKey的keys中
+ Set<String> keys = keywordParser.parse(value);
+ keys.add(contextKey.getKey());
+ contextValue.setKeywords(jackson.writeValueAsString(keys));
+ //3.缓存中是否有这个key,没有创建,有就更新
+ ContextKeyValue keyValue = contextCacheService.get(contextID, contextKey);
+ if (keyValue == null) {
+ //创建校验scope和tye
+ if (contextKey.getContextScope() == null || contextKey.getContextType() == null) {
+ throw new CSErrorException(97000, "try to create context ,type or scope cannot be empty");
+ }
+ //这里没有给出contextKeyvalue的具体实现,给个默认的persistence
+ logger.warn(String.format("setValueByKey, keyValue is not exist, csId:%s,key:%s", contextID.getContextId(), contextKey.getKey()));
+ keyValue = new PersistenceContextKeyValue();
+ keyValue.setContextKey(contextKey);
+ keyValue.setContextValue(contextValue);
+ getPersistence().create(contextID, keyValue);
+ } else {
+ //update的话,如果scope和type 是空的,要用数据库中的值,因为update缓存要用到
+ if (contextKey.getContextScope() == null) {
+ contextKey.setContextScope(keyValue.getContextKey().getContextScope());
+ }
+ if (contextKey.getContextType() == null) {
+ contextKey.setContextType(keyValue.getContextKey().getContextType());
+ }
+ keyValue.setContextKey(contextKey);
+ keyValue.setContextValue(contextValue);
+ getPersistence().update(contextID, keyValue);
+ }
+ //4.更新缓存
+ contextCacheService.put(contextID, keyValue);
+ logger.info(String.format("setValueByKey, csId:%s,key:%s,keywords:%s", contextID.getContextId(), contextKey.getKey(), contextValue.getKeywords()));
+ }
+
+ @Override
+ public void setValue(ContextID contextID, ContextKeyValue contextKeyValue) throws CSErrorException, ClassNotFoundException, JsonProcessingException {
+ //1.解析keywords
+ Object value = contextKeyValue.getContextValue().getValue();
+ Set<String> keys = keywordParser.parse(value);
+ keys.add(contextKeyValue.getContextKey().getKey());
+ contextKeyValue.getContextValue().setKeywords(jackson.writeValueAsString(keys));
+ //2.数据库中是否有这个key,没有就创建,有就更新
+ ContextKeyValue keyValue = contextCacheService.get(contextID, contextKeyValue.getContextKey());
+ if (keyValue == null) {
+ logger.warn("cache can not find contextId:{},key:{},now try to load from MySQL", contextID.getContextId(), contextKeyValue.getContextKey().getKey());
+ keyValue = getPersistence().get(contextID, contextKeyValue.getContextKey());
+ if (keyValue != null) {
+ logger.warn("MySQL find the key,now reset the cache and get it");
+ contextCacheService.rest(contextID, contextKeyValue.getContextKey());
+ keyValue = contextCacheService.get(contextID, contextKeyValue.getContextKey());
+ }
+ }
+ if (keyValue == null) {
+ if (contextKeyValue.getContextKey().getContextScope() == null || contextKeyValue.getContextKey().getContextType() == null) {
+ throw new CSErrorException(97000, "try to create context ,type or scope cannot be empty");
+ }
+ getPersistence().create(contextID, contextKeyValue);
+ } else {
+ //update的话,如果scope和type 是空的,要用数据库中的值,因为update缓存要用到
+ if (contextKeyValue.getContextKey().getContextScope() == null) {
+ contextKeyValue.getContextKey().setContextScope(keyValue.getContextKey().getContextScope());
+ }
+ if (contextKeyValue.getContextKey().getContextType() == null) {
+ contextKeyValue.getContextKey().setContextType(keyValue.getContextKey().getContextType());
+ }
+ getPersistence().update(contextID, contextKeyValue);
+ }
+ //3.更新缓存
+ contextCacheService.put(contextID, contextKeyValue);
+ logger.info(String.format("setValue, csId:%s,key:%s", contextID.getContextId(), contextKeyValue.getContextKey().getKey()));
+ }
+
+ @Override
+ public void resetValue(ContextID contextID, ContextKey contextKey) throws CSErrorException {
+ //1.reset 数据库
+ getPersistence().reset(contextID, contextKey);
+ //2.reset 缓存
+ contextCacheService.rest(contextID, contextKey);
+ logger.info(String.format("resetValue, csId:%s,key:%s", contextID.getContextId(), contextKey.getKey()));
+ }
+
+ @Override
+ public void removeValue(ContextID contextID, ContextKey contextKey) throws CSErrorException {
+ ContextKeyValue contextKeyValue = contextCacheService.get(contextID, contextKey);
+ if (contextKeyValue == null) {
+ return;
+ }
+ //如果scope和type 不为空,则从原数据中进行赋值
+ if (contextKey.getContextScope() == null) {
+ contextKey.setContextScope(contextKeyValue.getContextKey().getContextScope());
+ }
+ if (contextKey.getContextType() == null) {
+ contextKey.setContextType(contextKeyValue.getContextKey().getContextType());
+ }
+ //1.remove 数据库
+ getPersistence().remove(contextID, contextKey);
+ //2.remove 缓存
+ contextCacheService.remove(contextID, contextKey);
+ logger.info(String.format("removeValue, csId:%s,key:%s", contextID.getContextId(), contextKey.getKey()));
+ }
+
+ @Override
+ public void removeAllValue(ContextID contextID) throws CSErrorException {
+ //移除数据库
+ getPersistence().removeAll(contextID);
+ //移除缓存
+ contextCacheService.removeAll(contextID);
+
+ logger.info(String.format("removeAllValue, csId:%s", contextID.getContextId()));
+ }
+
+ @Override
+ public void removeAllValueByKeyPrefixAndContextType(ContextID contextID, ContextType contextType, String keyPrefix) throws CSErrorException {
+ contextCacheService.removeByKeyPrefix(contextID, keyPrefix, contextType);
+ getPersistence().removeByKeyPrefix(contextID, contextType, keyPrefix);
+ logger.info(String.format("removeAllValueByKeyPrefixAndContextType, csId:%s,csType:%s,keyPrefix:%s",
+ contextID.getContextId(), contextType, keyPrefix));
+ }
+
+ @Override
+ public void removeAllValueByKeyPrefix(ContextID contextID, String keyPrefix) throws CSErrorException {
+ contextCacheService.removeByKeyPrefix(contextID, keyPrefix);
+ getPersistence().removeByKeyPrefix(contextID, keyPrefix);
+ logger.info(String.format("removeAllValueByKeyPrefix, csId:%s,keyPrefix:%s",
+ contextID.getContextId(), keyPrefix));
+ }
+
+
+}
diff --git a/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/util/CsUtils.java b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/util/CsUtils.java
new file mode 100644
index 0000000..b82e5c4
--- /dev/null
+++ b/contextservice/cs-server/src/main/java/com/webank/wedatasphere/linkis/cs/server/util/CsUtils.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.cs.server.util;
+
+import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper;
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper;
+
+/**
+ * Created by patinousward on 2020/2/22.
+ */
+public class CsUtils {
+
+ public static final SerializationHelper SERIALIZE = ContextSerializationHelper.getInstance();
+
+ public static String serialize(Object o) throws CSErrorException {
+ if (o instanceof String){
+ return (String)o;
+ }
+ return SERIALIZE.serialize(o);
+ }
+
+}
diff --git a/contextservice/cs-server/src/main/resources/application.yml b/contextservice/cs-server/src/main/resources/application.yml
new file mode 100644
index 0000000..246846d
--- /dev/null
+++ b/contextservice/cs-server/src/main/resources/application.yml
@@ -0,0 +1,38 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+server:
+ port: 9004
+spring:
+ application:
+ name: cloud-contextservice
+
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://127.0.0.1:20303/eureka/
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
diff --git a/contextservice/cs-server/src/main/resources/linkis.properties b/contextservice/cs-server/src/main/resources/linkis.properties
new file mode 100644
index 0000000..9abef04
--- /dev/null
+++ b/contextservice/cs-server/src/main/resources/linkis.properties
@@ -0,0 +1,32 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.test.mode=true
+
+wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/ide_gz_bdap_sit_01?characterEncoding=UTF-8
+wds.linkis.server.mybatis.datasource.username=
+wds.linkis.server.mybatis.datasource.password=
+
+
+wds.linkis.log.clear=true
+wds.linkis.server.version=v1
+
+##restful
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.cs.server.restful
+
+##mybatis
+wds.linkis.server.mybatis.mapperLocations=classpath*:com\\webank\\wedatasphere\\linkis\\cs\\persistence\\dao\\impl\\*.xml
+
+wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.linkis.cs.persistence.entity
+
+wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.linkis.cs.persistence.dao
diff --git a/contextservice/cs-server/src/main/resources/log4j.properties b/contextservice/cs-server/src/main/resources/log4j.properties
new file mode 100644
index 0000000..0807e60
--- /dev/null
+++ b/contextservice/cs-server/src/main/resources/log4j.properties
@@ -0,0 +1,37 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
diff --git a/contextservice/cs-server/src/main/resources/log4j2.xml b/contextservice/cs-server/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..3923cd9
--- /dev/null
+++ b/contextservice/cs-server/src/main/resources/log4j2.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/contextservice/cs-ujes-client/pom.xml b/contextservice/cs-ujes-client/pom.xml
new file mode 100644
index 0000000..e0a03c2
--- /dev/null
+++ b/contextservice/cs-ujes-client/pom.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-cs-ujes-client</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-client</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-storage</artifactId>
+ <scope>provided</scope>
+ <version>${linkis.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/contextservice/cs-ujes-client/src/main/scala/com/webank/wedatasphere/linkis/cs/storage/CSTableResultSetWriter.scala b/contextservice/cs-ujes-client/src/main/scala/com/webank/wedatasphere/linkis/cs/storage/CSTableResultSetWriter.scala
new file mode 100644
index 0000000..03e8f50
--- /dev/null
+++ b/contextservice/cs-ujes-client/src/main/scala/com/webank/wedatasphere/linkis/cs/storage/CSTableResultSetWriter.scala
@@ -0,0 +1,74 @@
+/**
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.cs.storage
+
+import java.util.Date
+
+import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.client.service.CSTableService
+import com.webank.wedatasphere.linkis.cs.common.entity.metadata.{CSColumn, CSTable}
+import com.webank.wedatasphere.linkis.storage.resultset.StorageResultSetWriter
+import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord, TableResultSet}
+import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import org.apache.commons.lang.StringUtils
+
+/**
+ * @author peacewong
+ * @date 2020/4/1 21:03
+ */
+class CSTableResultSetWriter(tableResult: TableResultSet, maxCacheSize: Long,
+ storePath: FsPath, contextIDStr: String, nodeName: String, alias: String) extends StorageResultSetWriter[TableMetaData, TableRecord](tableResult, maxCacheSize, storePath) with Logging {
+
+ override def toString: String = {
+ try {
+ registerToCS
+ } catch {
+ case t: Throwable =>
+ info("Failed to register tmp table", t)
+ }
+ super.toString
+ }
+
+ private def registerToCS: Unit = {
+
+ if (StringUtils.isNotBlank(contextIDStr) && StringUtils.isNotBlank(nodeName) && !isEmpty) {
+ info("Start to register resultSet to cs")
+ flush()
+ val csTable = new CSTable
+ csTable.setAlias(alias)
+ csTable.setAvailable(true)
+ csTable.setComment("cs temp table")
+ csTable.setCreateTime(new Date())
+ csTable.setCreator(StorageUtils.getJvmUser)
+ csTable.setExternalUse(true)
+ csTable.setImport(false)
+ csTable.setLocation(toFSPath.getSchemaPath)
+ csTable.setPartitionTable(false)
+ csTable.setView(true)
+ val csColumns = getMetaData.asInstanceOf[TableMetaData].columns.map { column =>
+ val csColumn = new CSColumn
+ csColumn.setName(column.columnName)
+ csColumn.setType(column.dataType.typeName)
+ csColumn.setComment(column.comment)
+ csColumn
+ }
+ csTable.setColumns(csColumns)
+ CSTableService.getInstance().registerCSTable(contextIDStr, nodeName, alias, csTable)
+ info("Finished to register resultSet to cs")
+ }
+ }
+}
diff --git a/core/cloudModule/pom.xml b/core/cloudModule/pom.xml
index 576289b..aeaed0c 100644
--- a/core/cloudModule/pom.xml
+++ b/core/cloudModule/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -57,6 +57,18 @@
<artifactId>jsr311-api</artifactId>
<groupId>javax.ws.rs</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>jackson-annotations</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-core</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
</exclusions>
</dependency>
@@ -83,6 +95,22 @@
<artifactId>hibernate-validator</artifactId>
<groupId>org.hibernate.validator</groupId>
</exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-datatype-jdk8</artifactId>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-module-parameter-names</artifactId>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ </exclusion>
</exclusions>
<version>${spring.boot.version}</version>
</dependency>
@@ -112,6 +140,14 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ </exclusion>
</exclusions>
</dependency>
@@ -123,6 +159,14 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-annotations</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
</exclusions>
<version>2.0.0.RELEASE</version>
</dependency>
@@ -142,7 +186,7 @@
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
- <version>5.1.34</version>
+ <version>5.1.49</version>
</dependency>
<dependency>
@@ -242,6 +286,20 @@
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>jackson-annotations</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-core</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
@@ -300,6 +358,10 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
</exclusions>
</dependency>
diff --git a/core/cloudMybatis/pom.xml b/core/cloudMybatis/pom.xml
index 3b917bd..700ba3c 100644
--- a/core/cloudMybatis/pom.xml
+++ b/core/cloudMybatis/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
diff --git a/core/cloudProtocol/pom.xml b/core/cloudProtocol/pom.xml
index da11c8d..724286a 100644
--- a/core/cloudProtocol/pom.xml
+++ b/core/cloudProtocol/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestPersistTask.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestPersistTask.java
index 0db2c82..8bf25b1 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestPersistTask.java
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestPersistTask.java
@@ -26,17 +26,15 @@
/**
* Created by enjoyyin on 2018/9/30.
*/
-public class RequestPersistTask implements Task, QueryProtocol {
+public class RequestPersistTask implements Task {
private Long taskID;
/**
- * Instance is an instance of the unified entry where the task is located. ip + port
* instance 是指该task所在的统一入口的实例 ip + port
*/
private String instance;
private String execId;
private String umUser;
/**
- * EngineInstance is the instance information of the engine that the task executes, ip+port
* engineInstance 是指task执行所请求的engine的实例信息，ip+port
*/
private String engineInstance;
@@ -47,49 +45,72 @@
private String status;
private Date createdTime;
private Date updatedTime;
- private String engineType;
private Integer errCode;
private String errDesc;
+ private String taskResource;
/**
- * The executeApplicationName parameter refers to the service the user is looking for, such as spark python R, etc.
* executeApplicationName 参数指的是用户所寻求的服务，比如spark python R等等
*/
private String executeApplicationName;
/**
- * requestApplicationName is the name of the creator, such as IDE or WTSS
* requestApplicationName 是creator的传参名，例如IDE或WTSS等
*/
private String requestApplicationName;
/**
- * The user adopts the way of executing the script, and the scriptPath is the storage address of the script.
- * 用户采用传入执行脚本的方式，scriptPath就是脚本的存储地址
+ * source 存放脚本来源，scriptPath是其中一个参数用户采用传入执行脚本的方式，scriptPath就是脚本的存储地址
*/
- private String scriptPath;
- private java.util.Map<String, String> source;
+ private Map<String, String> source;
/**
- * The runType needs to be used in conjunction with the executeApplicationName. If the user selects Spark as the service, he also needs to specify which execution mode to use, such as pySpark RSpark.
- * runType and runType are the same attribute, in order to be compatible with the previous code
* runType需要和executeApplicationName结合使用，如用户选择了Spark做为服务，他还需要指明使用哪种执行方式，比如pySpark RSpark等
* runType和runType是同一个属性，为了兼容以前的代码
*/
private String runType;
- private java.util.Map<String, Object> params;
+ private String engineType;
+ private Map<String, Object> params;
+ private Date engineStartTime;
+ public String getEngineType() {
+ return engineType;
+ }
+ public void setEngineType(String engineType) {
+ this.engineType = engineType;
+ this.runType = engineType;
+ }
+
+ public Date getEngineStartTime() {
+ return engineStartTime;
+ }
+
+ public void setEngineStartTime(Date engineStartTime) {
+ this.engineStartTime = engineStartTime;
+ }
+
+ public String getRunType() {
+ return runType;
+ }
+
+ public void setRunType(String runType) {
+ this.runType = runType;
+ this.engineType = runType;
+ }
@Override
public String getInstance() {
return instance;
}
+
@Override
public String getExecId() {
return execId;
}
+
@Override
public void setInstance(String instance) {
this.instance = instance;
}
+
@Override
public void setExecId(String execId) {
this.execId = execId;
@@ -119,14 +140,6 @@
this.requestApplicationName = requestApplicationName;
}
- public String getScriptPath() {
- return scriptPath;
- }
-
- public void setScriptPath(String scriptPath) {
- this.scriptPath = scriptPath;
- }
-
public Map<String, String> getSource() {
return source;
}
@@ -135,15 +148,6 @@
this.source = source;
}
- public String getRunType() {
- return runType;
- }
-
- public void setRunType(String runType) {
- this.runType = runType;
- this.engineType = runType;
- }
-
public Long getTaskID() {
return taskID;
}
@@ -152,7 +156,6 @@
this.taskID = taskID;
}
-
public String getUmUser() {
return umUser;
}
@@ -161,8 +164,6 @@
this.umUser = umUser;
}
-
-
public Float getProgress() {
return progress;
}
@@ -203,15 +204,6 @@
this.updatedTime = updatedTime;
}
- public String getEngineType() {
- return engineType;
- }
-
- public void setEngineType(String engineType) {
- this.engineType = engineType;
- this.runType = engineType;
- }
-
public Integer getErrCode() {
return errCode;
}
@@ -232,7 +224,7 @@
return executionCode;
}
- public String getCode(){
+ public String getCode() {
return this.getExecutionCode();
}
@@ -256,6 +248,14 @@
this.engineInstance = engineInstance;
}
+ public String getTaskResource() {
+ return taskResource;
+ }
+
+ public void setTaskResource(String taskResource) {
+ this.taskResource = taskResource;
+ }
+
@Override
public String toString() {
return "RequestPersistTask{" +
@@ -271,36 +271,48 @@
", status='" + status + '\'' +
", createdTime=" + createdTime +
", updatedTime=" + updatedTime +
- ", engineType='" + engineType + '\'' +
", errCode=" + errCode +
", errDesc='" + errDesc + '\'' +
+ ", executeApplicationName='" + executeApplicationName + '\'' +
+ ", requestApplicationName='" + requestApplicationName + '\'' +
+ ", source=" + source +
+ ", runType='" + runType + '\'' +
+ ", params=" + params +
'}';
}
@Override
public boolean equals(Object o) {
- if (this == o) return true;
+ if (this == o) {
+ return true;
+ }
- if (o == null || getClass() != o.getClass()) return false;
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
- RequestPersistTask that = (RequestPersistTask) o;
+ RequestPersistTask task = (RequestPersistTask) o;
return new EqualsBuilder()
- .append(taskID, that.taskID)
- .append(instance, that.instance)
- .append(execId, that.execId)
- .append(umUser, that.umUser)
- .append(engineInstance, that.engineInstance)
- .append(executionCode, that.executionCode)
- .append(progress, that.progress)
- .append(logPath, that.logPath)
- .append(resultLocation, that.resultLocation)
- .append(status, that.status)
- .append(createdTime, that.createdTime)
- .append(updatedTime, that.updatedTime)
- .append(engineType, that.engineType)
- .append(errCode, that.errCode)
- .append(errDesc, that.errDesc)
+ .append(taskID, task.taskID)
+ .append(instance, task.instance)
+ .append(execId, task.execId)
+ .append(umUser, task.umUser)
+ .append(engineInstance, task.engineInstance)
+ .append(executionCode, task.executionCode)
+ .append(progress, task.progress)
+ .append(logPath, task.logPath)
+ .append(resultLocation, task.resultLocation)
+ .append(status, task.status)
+ .append(createdTime, task.createdTime)
+ .append(updatedTime, task.updatedTime)
+ .append(errCode, task.errCode)
+ .append(errDesc, task.errDesc)
+ .append(executeApplicationName, task.executeApplicationName)
+ .append(requestApplicationName, task.requestApplicationName)
+ .append(source, task.source)
+ .append(runType, task.runType)
+ .append(params, task.params)
.isEquals();
}
@@ -319,9 +331,13 @@
.append(status)
.append(createdTime)
.append(updatedTime)
- .append(engineType)
.append(errCode)
.append(errDesc)
+ .append(executeApplicationName)
+ .append(requestApplicationName)
+ .append(source)
+ .append(runType)
+ .append(params)
.toHashCode();
}
-}
+}
\ No newline at end of file
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java
index a03f072..886b75c 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/RequestReadAllTask.java
@@ -16,8 +16,6 @@
package com.webank.wedatasphere.linkis.protocol.query;
-import com.webank.wedatasphere.linkis.protocol.task.Task;
-
/**
* created by enjoyyin on 2018/10/9
* Description:
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java
index f3a2b27..39dde90 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/ResponsePersist.java
@@ -29,7 +29,7 @@
public class ResponsePersist implements QueryProtocol {
private Integer status;
private String msg;
- private Map<String,Object> data;
+ private Map<String, Object> data;
public Integer getStatus() {
return status;
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheNotFound.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheNotFound.java
new file mode 100644
index 0000000..f0c719e
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheNotFound.java
@@ -0,0 +1,4 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+public class CacheNotFound implements ResponseReadCache {
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheTaskResult.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheTaskResult.java
new file mode 100644
index 0000000..7f36f7d
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/CacheTaskResult.java
@@ -0,0 +1,14 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+public class CacheTaskResult implements ResponseReadCache {
+
+ private String resultLocation;
+
+ public CacheTaskResult(String resultLocation) {
+ this.resultLocation = resultLocation;
+ }
+
+ public String getResultLocation() {
+ return resultLocation;
+ }
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/FailedToDeleteCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/FailedToDeleteCache.java
new file mode 100644
index 0000000..41cdca6
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/FailedToDeleteCache.java
@@ -0,0 +1,13 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+public class FailedToDeleteCache {
+ private String errorMessage;
+
+ public FailedToDeleteCache(String errorMessage) {
+ this.errorMessage = errorMessage;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestDeleteCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestDeleteCache.java
new file mode 100644
index 0000000..317eed7
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestDeleteCache.java
@@ -0,0 +1,28 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
+
+public class RequestDeleteCache implements QueryProtocol {
+
+ private String executionCode;
+ private String engineType;
+ private String user;
+
+ public RequestDeleteCache(String executionCode, String engineType, String user) {
+ this.executionCode = executionCode;
+ this.engineType = engineType;
+ this.user = user;
+ }
+
+ public String getExecutionCode() {
+ return executionCode;
+ }
+
+ public String getEngineType() {
+ return engineType;
+ }
+
+ public String getUser() {
+ return user;
+ }
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestReadCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestReadCache.java
new file mode 100644
index 0000000..49ed99e
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/RequestReadCache.java
@@ -0,0 +1,33 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
+
+public class RequestReadCache implements QueryProtocol {
+ private String executionCode;
+ private String engineType;
+ private String user;
+ private Long readCacheBefore;
+
+ public RequestReadCache(String executionCode, String engineType, String user, Long readCacheBefore) {
+ this.executionCode = executionCode;
+ this.engineType = engineType;
+ this.user = user;
+ this.readCacheBefore = readCacheBefore;
+ }
+
+ public String getExecutionCode() {
+ return executionCode;
+ }
+
+ public String getEngineType() {
+ return engineType;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public Long getReadCacheBefore() {
+ return readCacheBefore;
+ }
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseDeleteCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseDeleteCache.java
new file mode 100644
index 0000000..e9c0e3a
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseDeleteCache.java
@@ -0,0 +1,6 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
+
+public interface ResponseDeleteCache extends QueryProtocol {
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseReadCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseReadCache.java
new file mode 100644
index 0000000..dbcc84a
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/ResponseReadCache.java
@@ -0,0 +1,6 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+import com.webank.wedatasphere.linkis.protocol.query.QueryProtocol;
+
+public interface ResponseReadCache extends QueryProtocol {
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/SuccessDeletedCache.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/SuccessDeletedCache.java
new file mode 100644
index 0000000..b16098f
--- /dev/null
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/query/cache/SuccessDeletedCache.java
@@ -0,0 +1,4 @@
+package com.webank.wedatasphere.linkis.protocol.query.cache;
+
+public class SuccessDeletedCache {
+}
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java
index 52bde81..f36a021 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/RequestRegister.java
@@ -16,8 +16,6 @@
package com.webank.wedatasphere.linkis.protocol.usercontrol;
-import java.util.Date;
-
/**
* Created by alexyang
*/
diff --git a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java
index b2b4217..5c12d62 100644
--- a/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java
+++ b/core/cloudProtocol/src/main/java/com/webank/wedatasphere/linkis/protocol/usercontrol/ResponseRegister.java
@@ -21,7 +21,7 @@
/**
* Created by alexyang
*/
-public class ResponseRegister implements UserControlRegtisterProtocol{
+public class ResponseRegister implements UserControlRegtisterProtocol {
private int status;
private String message;
diff --git a/core/cloudRPC/pom.xml b/core/cloudRPC/pom.xml
index e381f96..1e44677 100644
--- a/core/cloudRPC/pom.xml
+++ b/core/cloudRPC/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/conf/RPCConfiguration.scala b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/conf/RPCConfiguration.scala
index 8315b3e..e1d0670 100644
--- a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/conf/RPCConfiguration.scala
+++ b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/conf/RPCConfiguration.scala
@@ -39,4 +39,7 @@
val PUBLIC_SERVICE_APPLICATION_NAME = CommonVars("wds.linkis.gateway.conf.publicservice.name", "publicservice")
val PUBLIC_SERVICE_LIST = CommonVars("wds.linkis.gateway.conf.publicservice.list", "query,jobhistory,application,configuration,filesystem,udf,variable").getValue.split(",")
+ val BDP_RPC_INSTANCE_ALIAS_SERVICE_REFRESH_INTERVAL = CommonVars("wds.linkis.rpc.instancealias.refresh.interval", new TimeType("3s"))
+
+ val CONTEXT_SERVICE_APPLICATION_NAME = CommonVars("wds.linkis.gateway.conf.contextservice.name", "contextservice")
}
diff --git a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/InstanceAliasConverter.scala b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/InstanceAliasConverter.scala
new file mode 100644
index 0000000..3b14808
--- /dev/null
+++ b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/InstanceAliasConverter.scala
@@ -0,0 +1,14 @@
+package com.webank.wedatasphere.linkis.rpc.instancealias
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/18
+ */
+trait InstanceAliasConverter {
+
+ def instanceToAlias(instance: String): String
+
+ def aliasToInstance(alias: String): String
+
+ def checkAliasFormatValid(alias: String): Boolean
+}
diff --git a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/InstanceAliasManager.scala b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/InstanceAliasManager.scala
new file mode 100644
index 0000000..3a6b4a7
--- /dev/null
+++ b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/InstanceAliasManager.scala
@@ -0,0 +1,24 @@
+package com.webank.wedatasphere.linkis.rpc.instancealias
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import javax.annotation.Nullable
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/18
+ */
+trait InstanceAliasManager {
+
+ def getAliasByServiceInstance(instance: ServiceInstance): String
+
+ def getAliasByInstance(instance: String): String
+
+ @Nullable
+ def getInstanceByAlias(alias: String): ServiceInstance
+
+ def refresh(): Unit
+
+ def getAllInstanceList(): java.util.List[ServiceInstance]
+
+ def isInstanceAliasValid(alias: String): Boolean
+}
diff --git a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/impl/DefaultInstanceAliasConverter.scala b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/impl/DefaultInstanceAliasConverter.scala
new file mode 100644
index 0000000..2957cdf
--- /dev/null
+++ b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/impl/DefaultInstanceAliasConverter.scala
@@ -0,0 +1,35 @@
+package com.webank.wedatasphere.linkis.rpc.instancealias.impl
+
+import java.util.Base64
+import java.util.regex.Pattern
+
+import com.webank.wedatasphere.linkis.rpc.instancealias.InstanceAliasConverter
+import org.apache.commons.lang.StringUtils
+import org.springframework.stereotype.Component
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/18
+ */
+@Component
+class DefaultInstanceAliasConverter extends InstanceAliasConverter {
+
+ val pattern = Pattern.compile("[a-zA-Z\\d=\\+/]+")
+
+ // todo use base64 for the moment
+ override def instanceToAlias(instance: String): String = {
+ new String(Base64.getEncoder.encode(instance.getBytes()))
+ }
+
+ override def aliasToInstance(alias: String): String = {
+ new String(Base64.getDecoder.decode(alias))
+ }
+
+ override def checkAliasFormatValid(alias: String): Boolean = {
+ if (StringUtils.isBlank(alias)) {
+ return false
+ }
+ val matcher = pattern.matcher(alias)
+ matcher.find()
+ }
+}
diff --git a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/impl/InstanceAliasManagerImpl.scala b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/impl/InstanceAliasManagerImpl.scala
new file mode 100644
index 0000000..353f6a4
--- /dev/null
+++ b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/instancealias/impl/InstanceAliasManagerImpl.scala
@@ -0,0 +1,92 @@
+package com.webank.wedatasphere.linkis.rpc.instancealias.impl
+
+import java.util
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.rpc.conf.RPCConfiguration
+import com.webank.wedatasphere.linkis.rpc.instancealias.{InstanceAliasConverter, InstanceAliasManager}
+import com.webank.wedatasphere.linkis.rpc.sender.eureka.EurekaRPCServerLoader
+import com.webank.wedatasphere.linkis.rpc.utils.RPCUtils
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Component
+
+import scala.collection.JavaConversions._
+
+/**
+ * @Author alexyang
+ * @Date 2020/2/18
+ */
+@Component
+class InstanceAliasManagerImpl extends InstanceAliasManager with Logging {
+
+ private val serverLoader = new EurekaRPCServerLoader()
+
+ private val mainInstance = DataWorkCloudApplication.getServiceInstance
+
+ @Autowired
+ var instanceAliasConverter: InstanceAliasConverter = _
+
+ override def getAliasByInstance(instance: String): String = {
+ instanceAliasConverter.instanceToAlias(instance)
+ }
+
+ override def getInstanceByAlias(alias: String): ServiceInstance = {
+ val serviceID = getContextServiceID()
+ if (null == serviceID) {
+ return null
+ }
+ val instances = serverLoader.getServiceInstances(serviceID)
+ if (null == instances || instances.isEmpty) {
+ error(s"None serviec instances for Context Service ID : " + serviceID)
+ return null
+ }
+ val targetInstance = instanceAliasConverter.aliasToInstance(alias)
+ instances.foreach(ins => {
+ if (ins.getInstance.equalsIgnoreCase(targetInstance)) {
+ return ins
+ }
+ })
+ null
+ }
+
+ def getContextServiceID(): String = {
+ RPCUtils.findService (RPCConfiguration.CONTEXT_SERVICE_APPLICATION_NAME.getValue, list => {
+ val services = list.filter (_.contains (RPCConfiguration.CONTEXT_SERVICE_APPLICATION_NAME.getValue) )
+ services.headOption
+ }).getOrElse(null)
+ }
+
+ @Deprecated
+ override def refresh(): Unit = {
+
+ }
+
+ override def getAllInstanceList(): util.List[ServiceInstance] = {
+ val serviceID = getContextServiceID()
+ if (null == serviceID) {
+ return new util.ArrayList[ServiceInstance](0)
+ }
+ serverLoader.getServiceInstances(serviceID).toList
+ }
+
+ override def isInstanceAliasValid(alias: String): Boolean = {
+ if (!instanceAliasConverter.checkAliasFormatValid(alias)) {
+ return false
+ }
+ if (null != getInstanceByAlias(alias)) {
+ true
+ } else {
+ false
+ }
+ }
+
+ override def getAliasByServiceInstance(instance: ServiceInstance): String = {
+ if (null == instance) {
+ return null
+ }
+ instanceAliasConverter.instanceToAlias(instance.getInstance)
+ }
+}
+
diff --git a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/utils/RPCUtils.scala b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/utils/RPCUtils.scala
index cbaa838..cd50461 100644
--- a/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/utils/RPCUtils.scala
+++ b/core/cloudRPC/src/main/scala/com/webank/wedatasphere/linkis/rpc/utils/RPCUtils.scala
@@ -21,8 +21,10 @@
import com.netflix.client.ClientException
import com.webank.wedatasphere.linkis.rpc.exception.NoInstanceExistsException
+import com.webank.wedatasphere.linkis.rpc.sender.SpringCloudFeignConfigurationCache
import feign.RetryableException
import org.apache.commons.lang.StringUtils
+import scala.collection.JavaConversions._
/**
* Created by enjoyyin on 2019/2/22.
@@ -50,4 +52,11 @@
case _ => false
}
+ def findService(parsedServiceId: String, tooManyDeal: List[String] => Option[String]): Option[String] = {
+ val services = SpringCloudFeignConfigurationCache.getDiscoveryClient
+ .getServices.filter(_.toLowerCase.contains(parsedServiceId.toLowerCase)).toList
+ if(services.length == 1) Some(services.head)
+ else if(services.length > 1) tooManyDeal(services)
+ else None
+ }
}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index c4c7998..68b9b1d 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -78,7 +78,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
- <version>2.9.5</version>
+ <version>${fasterxml.jackson.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
@@ -96,6 +96,25 @@
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-parameter-names</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jsr310</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.datatype</groupId>
+ <artifactId>jackson-datatype-jdk8</artifactId>
+ <version>${fasterxml.jackson.version}</version>
+ </dependency>
+
+
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
diff --git a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/Configuration.scala b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/Configuration.scala
index 0053a17..9b9efbd 100644
--- a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/Configuration.scala
+++ b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/Configuration.scala
@@ -16,10 +16,12 @@
package com.webank.wedatasphere.linkis.common.conf
+import com.webank.wedatasphere.linkis.common.utils.Logging
+
/**
* Created by enjoyyin on 2018/4/18.
*/
-object Configuration {
+object Configuration extends Logging {
val BDP_ENCODING = CommonVars("wds.linkis.encoding", "utf-8")
@@ -31,4 +33,17 @@
val hadoopConfDir = CommonVars("hadoop.config.dir", CommonVars("HADOOP_CONF_DIR", "").getValue).getValue
+ val GATEWAY_URL: CommonVars[String] = CommonVars[String]("wds.linkis.gateway.url", "http://localhost:9001/")
+
+ def getGateWayURL(): String = {
+ val url = GATEWAY_URL.getValue.trim
+ val gatewayUr = if (url.endsWith("/")) {
+ url.substring(0, url.length - 1)
+ } else {
+ url
+ }
+ info(s"gatewayUrl is $gatewayUr")
+ gatewayUr
+ }
+
}
diff --git a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/listener/ListenerEventBus.scala b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/listener/ListenerEventBus.scala
index da3a325..035fd66 100644
--- a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/listener/ListenerEventBus.scala
+++ b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/listener/ListenerEventBus.scala
@@ -17,7 +17,7 @@
package com.webank.wedatasphere.linkis.common.listener
import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong}
-import java.util.concurrent.{CopyOnWriteArrayList, Future, TimeoutException}
+import java.util.concurrent.{ArrayBlockingQueue, CopyOnWriteArrayList, Future, LinkedBlockingQueue, TimeoutException}
import com.webank.wedatasphere.linkis.common.collection.BlockingLoopArray
import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging, Utils}
@@ -85,7 +85,7 @@
// protected val listenerConsumerThreadSize: Int = 5
// protected val listenerThreadMaxFreeTime: Long = ByteTimeUtils.timeStringAsMs("2m")
- private lazy val eventQueue = new BlockingLoopArray[E](eventQueueCapacity)
+ private lazy val eventQueue = new ArrayBlockingQueue[E](eventQueueCapacity)
protected val executorService = Utils.newCachedThreadPool(listenerConsumerThreadSize + 2, name + "-Consumer-ThreadPool", true)
private val eventDealThreads = Array.tabulate(listenerConsumerThreadSize)(new ListenerEventThread(_))
private val started = new AtomicBoolean(false)
@@ -160,7 +160,7 @@
* The use of synchronized here guarantees that all events that once belonged to this queue
* have already been processed by all attached listeners, if this returns true.
*/
- private def queueIsEmpty: Boolean = synchronized { !eventQueue.nonEmpty && !eventDealThreads.exists(_.isRunning) }
+ private def queueIsEmpty: Boolean = synchronized { eventQueue.isEmpty && !eventDealThreads.exists(_.isRunning) }
/**
* Stop the listener bus. It will wait until the queued events have been processed, but drop the
diff --git a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/utils/ClassUtils.scala b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/utils/ClassUtils.scala
index 076aec2..1075486 100644
--- a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/utils/ClassUtils.scala
+++ b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/utils/ClassUtils.scala
@@ -31,7 +31,29 @@
}
}
- def getClassInstance[T](className: String): T ={
+ def getClassInstance[T](className: String): T = {
Thread.currentThread.getContextClassLoader.loadClass(className).asInstanceOf[Class[T]].newInstance()
}
+
+ def getFieldVal(o: Any, name: String): Any = {
+ Utils.tryThrow {
+ val field = o.getClass.getDeclaredField(name)
+ field.setAccessible(true)
+ field.get(o)
+ } {
+ case t: Throwable => throw t
+ }
+ }
+
+ def setFieldVal(o: Any, name: String, value: Any): Unit = {
+ Utils.tryThrow {
+ val field = o.getClass.getDeclaredField(name)
+ field.setAccessible(true)
+ field.set(o, value.asInstanceOf[AnyRef])
+ } {
+ case t: Throwable => throw t
+ }
+ }
+
+
}
diff --git a/core/hadoop-common/pom.xml b/core/hadoop-common/pom.xml
index a691d8d..9e80c45 100644
--- a/core/hadoop-common/pom.xml
+++ b/core/hadoop-common/pom.xml
@@ -1,11 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -156,6 +169,14 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/conf/HadoopConf.scala b/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/conf/HadoopConf.scala
index 8ed9b16..24e10df 100644
--- a/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/conf/HadoopConf.scala
+++ b/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/conf/HadoopConf.scala
@@ -1,3 +1,16 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package com.webank.wedatasphere.linkis.hadoop.common.conf
import com.webank.wedatasphere.linkis.common.conf.CommonVars
diff --git a/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/utils/HDFSUtils.scala b/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/utils/HDFSUtils.scala
index 0922a3d..3982b1a 100644
--- a/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/utils/HDFSUtils.scala
+++ b/core/hadoop-common/src/main/scala/com/webank/wedatasphere/linkis/hadoop/common/utils/HDFSUtils.scala
@@ -1,14 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.webank.wedatasphere.linkis.hadoop.common.utils
import java.io.File
import java.nio.file.Paths
import java.security.PrivilegedExceptionAction
+import com.webank.wedatasphere.linkis.common.conf.Configuration.hadoopConfDir
+import com.webank.wedatasphere.linkis.hadoop.common.conf.HadoopConf._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
-import com.webank.wedatasphere.linkis.hadoop.common.conf.HadoopConf._
import org.apache.hadoop.security.UserGroupInformation
-import com.webank.wedatasphere.linkis.common.conf.Configuration.hadoopConfDir
/**
* Created by enjoyyin on 2019/5/27.
*/
diff --git a/core/httpclient/pom.xml b/core/httpclient/pom.xml
index 6b501b1..185dd85 100644
--- a/core/httpclient/pom.xml
+++ b/core/httpclient/pom.xml
@@ -1,13 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2019 WeBank
- ~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
- ~
~ http://www.apache.org/licenses/LICENSE-2.0
- ~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,18 +18,17 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>linkis-httpclient</artifactId>
- <packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis-common</artifactId>
- <scope>provided</scope>
+ <version>${linkis.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
@@ -40,6 +36,12 @@
<version>${httpclient.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpmime.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
<version>${json4s.version}</version>
@@ -63,35 +65,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>net.databinder.dispatch</groupId>
- <artifactId>dispatch-core_${scala.binary.version}</artifactId>
- <version>${dispatch.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>net.databinder.dispatch</groupId>
- <artifactId>dispatch-json4s-jackson_${scala.binary.version}</artifactId>
- <version>${dispatch.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- <version>${jetty.version}</version>
- </dependency>
</dependencies>
<build>
diff --git a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/AbstractHttpClient.scala b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/AbstractHttpClient.scala
index 0bdc4d9..e610b63 100644
--- a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/AbstractHttpClient.scala
+++ b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/AbstractHttpClient.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,11 +13,9 @@
package com.webank.wedatasphere.linkis.httpclient
-import java.io.{File, InputStream}
+import java.util
import java.util.concurrent.TimeUnit
-import com.ning.http.client.Response
-import com.ning.http.multipart.{FilePart, PartSource, StringPart}
import com.webank.wedatasphere.linkis.common.conf.Configuration
import com.webank.wedatasphere.linkis.common.io.{Fs, FsPath}
import com.webank.wedatasphere.linkis.common.utils.Utils
@@ -31,17 +26,25 @@
import com.webank.wedatasphere.linkis.httpclient.loadbalancer.{AbstractLoadBalancer, DefaultLoadbalancerStrategy, LoadBalancer}
import com.webank.wedatasphere.linkis.httpclient.request._
import com.webank.wedatasphere.linkis.httpclient.response._
-import dispatch._
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils
-import org.apache.http.HttpException
+import org.apache.http.client.entity.UrlEncodedFormEntity
+import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost}
+import org.apache.http.client.utils.URIBuilder
+import org.apache.http.entity.mime.MultipartEntityBuilder
+import org.apache.http.entity.{ContentType, StringEntity}
+import org.apache.http.impl.client.HttpClients
+import org.apache.http.message.BasicNameValuePair
+import org.apache.http.util.EntityUtils
+import org.apache.http.{HttpException, HttpResponse, _}
import org.json4s.jackson.Serialization.read
import org.json4s.{DefaultFormats, Formats}
import scala.collection.Iterable
import scala.collection.JavaConversions._
import scala.concurrent.duration.Duration
-import scala.concurrent.{Await, ExecutionContext,ExecutionContextExecutorService, Future}
+import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService}
+
/**
* Created by enjoyyin on 2019/5/20.
@@ -51,9 +54,11 @@
protected implicit val formats: Formats = DefaultFormats
protected implicit val executors: ExecutionContext = Utils.newCachedExecutionContext(clientConfig.getMaxConnection, clientName, false)
protected val httpTimeout: Duration = if (clientConfig.getReadTimeout > 0) Duration(clientConfig.getReadTimeout, TimeUnit.MILLISECONDS)
- else Duration.Inf
+ else Duration.Inf
- if(clientConfig.getAuthenticationStrategy != null) clientConfig.getAuthenticationStrategy match {
+ protected val httpClient = HttpClients.createDefault()
+
+ if (clientConfig.getAuthenticationStrategy != null) clientConfig.getAuthenticationStrategy match {
case auth: AbstractAuthenticationStrategy => auth.setClient(this)
case _ =>
}
@@ -70,8 +75,8 @@
//如果discovery没有启用，那么启用loadBalancer是没有意义的
val loadBalancer = if (clientConfig.isLoadbalancerEnabled && this.clientConfig.getLoadbalancerStrategy != null)
Some(this.clientConfig.getLoadbalancerStrategy.createLoadBalancer())
- else if(clientConfig.isLoadbalancerEnabled) Some(DefaultLoadbalancerStrategy.createLoadBalancer())
- else None
+ else if (clientConfig.isLoadbalancerEnabled) Some(DefaultLoadbalancerStrategy.createLoadBalancer())
+ else None
loadBalancer match {
case Some(lb: AbstractLoadBalancer) =>
discovery.foreach(_.addDiscoveryListener(lb))
@@ -90,20 +95,18 @@
if(!requestAction.isInstanceOf[HttpAction])
throw new UnsupportedOperationException("only HttpAction supported, but the fact is " + requestAction.getClass)
val action = prepareAction(requestAction.asInstanceOf[HttpAction])
- val req = prepareReq(action)
- val response = if(!clientConfig.isRetryEnabled) executeRequest(req, Some(waitTime).filter(_ > 0))
- else clientConfig.getRetryHandler.retry(executeRequest(req, Some(waitTime).filter(_ > 0)), action.getClass.getSimpleName + "HttpRequest")
+ val response: CloseableHttpResponse = executeHttpAction(action)
responseToResult(response, action)
}
override def execute(requestAction: Action, resultListener: ResultListener): Unit = {
- if(!requestAction.isInstanceOf[HttpAction])
+ if (!requestAction.isInstanceOf[HttpAction]) {
throw new UnsupportedOperationException("only HttpAction supported, but the fact is " + requestAction.getClass)
+ }
val action = prepareAction(requestAction.asInstanceOf[HttpAction])
- val req = prepareReq(action)
- val response = executeAsyncRequest(req)
- response.onSuccess{case r => resultListener.onSuccess(responseToResult(r, action))}
- response.onFailure{case t => resultListener.onFailure(t)}
+ val response: CloseableHttpResponse = executeHttpAction(action)
+ //response.onSuccess{case r => resultListener.onSuccess(responseToResult(r, action))}
+ //response.onFailure{case t => resultListener.onFailure(t)}
}
protected def getRequestUrl(suffixUrl: String, requestBody: String): String = {
@@ -121,93 +124,113 @@
protected def prepareAction(requestAction: HttpAction): HttpAction = requestAction
- protected def prepareReq(requestAction: HttpAction): Req = {
+ protected def executeHttpAction(requestAction: HttpAction): CloseableHttpResponse = {
var realURL = ""
- var req = requestAction match {
+ requestAction match {
case serverUrlAction: ServerUrlAction =>
- realURL = serverUrlAction.serverUrl
- dispatch.url(connectUrl(serverUrlAction.serverUrl, requestAction.getURL))
+ realURL = connectUrl(serverUrlAction.serverUrl, requestAction.getURL)
case _ =>
- val url = getRequestUrl(requestAction.getURL, requestAction.getRequestBody)
- realURL = url.replaceAll(requestAction.getURL, "")
- dispatch.url(url)
+ realURL = getRequestUrl(requestAction.getURL, requestAction.getRequestBody)
}
- var request = requestAction match {
- case upload: UploadAction =>
- req = req.setContentType("multipart/form-data", Configuration.BDP_ENCODING.getValue).POST
- if(upload.files != null && upload.files.nonEmpty) {
- val fs = upload.user.map(getFsByUser(_, new FsPath(upload.files.head._2)))
- upload.files.foreach { case (k, v) =>
- if(StringUtils.isEmpty(v)) throw new HttpException(s"$k 的文件路径不能为空！")
- val filePart = fs.map(f => if(f.exists(new FsPath(v))) new FilePart(k, new FsPartSource(f, v))
- else throw new HttpException(s"File $v 不存在！")).getOrElse{
- val f = new File(v)
- if(!f.exists() || !f.isFile) throw new HttpException(s"File $v 不存在！")
- else new FilePart(k, f)
- }
- req = req.addBodyPart(filePart)
- }
- }
- if(upload.inputStreams != null)
- upload.inputStreams.foreach { case (k, v) =>
- val filePart = new FilePart(k, new PartSource{
- val length = v.available
- override def getLength: Long = length
- override def getFileName: String = upload.inputStreamNames.getOrDefault(k, k)
- override def createInputStream(): InputStream = v
- })
- req = req.addBodyPart(filePart)
- }
- upload match {
- case get: GetAction => get.getParameters.foreach { case (k, v) => req = req.addBodyPart(new StringPart(k, v.toString)) }
- case _ =>
- }
- req
- case post: POSTAction =>
- req = req.POST
- if(!post.getParameters.isEmpty) post.getParameters.foreach{ case (k, v) => req = req.addQueryParameter(k, v.toString)}
- if(post.getFormParams.nonEmpty) {
- req.setContentType("application/x-www-form-urlencoded", Configuration.BDP_ENCODING.getValue) << post.getFormParams
- } else req.setContentType("application/json", Configuration.BDP_ENCODING.getValue) << post.getRequestPayload
- case get: GetAction =>
- if(!get.getParameters.isEmpty) get.getParameters.foreach{ case (k, v) => req = req.addQueryParameter(k, v.toString)}
- req.GET
- case _ =>
- req.POST.setBody(requestAction.getRequestBody)
- .setContentType("application/json", Configuration.BDP_ENCODING.getValue)
- }
- if(clientConfig.getAuthenticationStrategy != null) clientConfig.getAuthenticationStrategy.login(requestAction, realURL) match {
+
+ if (clientConfig.getAuthenticationStrategy != null) clientConfig.getAuthenticationStrategy.login(requestAction, realURL.replaceAll(requestAction.getURL, "")) match {
case authAction: HttpAuthentication =>
val cookies = authAction.authToCookies
- if(cookies != null && cookies.nonEmpty) cookies.foreach(requestAction.addCookie)
+ if (cookies != null && cookies.nonEmpty) cookies.foreach(requestAction.addCookie)
val headers = authAction.authToHeaders
- if(headers != null && headers.nonEmpty) headers.foreach{case (k, v) => requestAction.addHeader(k, v.toString)}
+ if (headers != null && !headers.isEmpty()) {
+ headers.foreach { case (k, v) => requestAction.addHeader(k.toString(), v.toString()) }
+ }
case _ =>
}
- if(requestAction.getHeaders.nonEmpty) request = request <:< requestAction.getHeaders
- if(requestAction.getCookies.nonEmpty) requestAction.getCookies.foreach(c => request = request.addCookie(c))
- request
+
+ var response: CloseableHttpResponse = null
+ requestAction match {
+ case upload: UploadAction =>
+ val httpPost = new HttpPost(realURL)
+ val builder = MultipartEntityBuilder.create()
+ if(upload.inputStreams != null)
+ upload.inputStreams.foreach { case (k, v) =>
+ builder.addBinaryBody(k, v, ContentType.create("multipart/form-data"), k)
+ }
+ upload match {
+ case get: GetAction => get.getParameters.
+ retain((k, v) => v != null && k != null).
+ foreach { case (k, v) => builder.addTextBody(k.toString, v.toString) }
+ case _ =>
+ }
+ upload match {
+ case get: GetAction => get.getHeaders.
+ retain((k, v) => v != null && k != null).
+ foreach { case (k, v) => httpPost.addHeader(k.toString, v.toString) }
+ case _ =>
+ }
+ val httpEntity = builder.build()
+ httpPost.setEntity(httpEntity)
+ response = httpClient.execute(httpPost)
+ case post: POSTAction =>
+ val httpPost = new HttpPost(realURL)
+ if (post.getParameters.nonEmpty || post.getFormParams.nonEmpty) {
+ val nvps = new util.ArrayList[NameValuePair]
+ if (post.getParameters.nonEmpty) {
+ post.getParameters.foreach { case (k, v) => nvps.add(new BasicNameValuePair(k, v.toString())) }
+ }
+ if (post.getFormParams.nonEmpty) {
+ post.getFormParams.foreach { case (k, v) => nvps.add(new BasicNameValuePair(k, v.toString())) }
+ }
+ httpPost.setEntity(new UrlEncodedFormEntity(nvps))
+ }
+
+ if (StringUtils.isNotBlank(post.getRequestPayload)) {
+ val stringEntity = new StringEntity(post.getRequestPayload, "UTF-8")
+ stringEntity.setContentEncoding(Configuration.BDP_ENCODING.getValue)
+ stringEntity.setContentType("application/json")
+ httpPost.setEntity(stringEntity)
+ }
+
+ if (requestAction.getHeaders.nonEmpty) {
+ requestAction.getHeaders.foreach { case (k, v) => httpPost.addHeader(k.toString(), v.toString()) }
+ }
+ response = httpClient.execute(httpPost)
+ case get: GetAction =>
+ val builder = new URIBuilder(realURL)
+ if (!get.getParameters.isEmpty) {
+ get.getParameters.foreach { case (k, v) => builder.addParameter(k.toString(), v.toString()) }
+ }
+ val httpGet = new HttpGet(builder.build())
+ if (requestAction.getHeaders.nonEmpty) {
+ requestAction.getHeaders.foreach { case (k, v) => httpGet.addHeader(k.toString(), v.toString()) }
+ }
+ response = httpClient.execute(httpGet);
+ case _ =>
+ val httpost = new HttpPost(realURL)
+ val stringEntity = new StringEntity(requestAction.getRequestBody, "UTF-8")
+ stringEntity.setContentEncoding(Configuration.BDP_ENCODING.getValue)
+ stringEntity.setContentType("application/json")
+ httpost.setEntity(stringEntity)
+ if (requestAction.getHeaders.nonEmpty) {
+ requestAction.getHeaders.foreach { case (k, v) => httpost.addHeader(k.toString(), v.toString()) }
+ }
+ response = httpClient.execute(httpost)
+ }
+ response
}
protected def getFsByUser(user: String, path: FsPath): Fs
- protected def executeRequest(req: Req, waitTime: Option[Long]): Response = {
- val response = executeAsyncRequest(req)
- Await.result(response, Duration(waitTime.getOrElse(clientConfig.getReadTimeout), TimeUnit.MILLISECONDS))
- }
-
- protected def executeAsyncRequest(req: Req): Future[Response] = Http(req > as.Response(r => r))
-
- protected def responseToResult(response: Response, requestAction: Action): Result = {
+ protected def responseToResult(response: HttpResponse, requestAction: Action): Result = {
+ var entity = response.getEntity
val result = requestAction match {
case download: DownloadAction =>
- val statusCode = response.getStatusCode
- if(statusCode != 200) {
- val responseBody = response.getResponseBody
- val url = response.getUri.toString
- throw new HttpClientResultException(s"URL $url request failed! ResponseBody is $responseBody." )
+ val statusCode = response.getStatusLine.getStatusCode
+ if (statusCode != 200) {
+ var responseBody: String = null
+ if (entity != null) {
+ responseBody = EntityUtils.toString(entity, "UTF-8")
+ }
+ throw new HttpClientResultException(s"request failed! ResponseBody is $responseBody.")
}
- download.write(response.getResponseBodyAsStream)
+ download.write(entity.getContent)
Result()
case heartbeat: HeartbeatAction =>
discovery.map {
@@ -219,8 +242,12 @@
case _ => throw new HttpMessageParseException("AuthenticationStrategy is not enable, login is not needed!")
}
case httpAction: HttpAction =>
- httpResponseToResult(response, httpAction)
- .getOrElse(throw new HttpMessageParseException("cannot parse message: " + response.getResponseBody))
+ var responseBody: String = null
+ if (entity != null) {
+ responseBody = EntityUtils.toString(entity, "UTF-8")
+ }
+ httpResponseToResult(response, httpAction, responseBody)
+ .getOrElse(throw new HttpMessageParseException("cannot parse message: " + responseBody))
}
result match {
case userAction: UserAction => requestAction match {
@@ -232,34 +259,29 @@
result
}
- protected def httpResponseToResult(response: Response, requestAction: HttpAction): Option[Result]
+ protected def httpResponseToResult(response: HttpResponse, requestAction: HttpAction, responseBody: String): Option[Result]
- protected def deserializeResponseBody(response: Response): Iterable[_] = {
- val responseBody = response.getResponseBody
- if(responseBody.startsWith("{") && responseBody.endsWith("}"))
+ protected def deserializeResponseBody(response: HttpResponse): Iterable[_] = {
+ var entity = response.getEntity
+ var responseBody: String = null
+ if (entity != null) {
+ responseBody = EntityUtils.toString(entity, "UTF-8")
+ }
+ if (responseBody.startsWith("{") && responseBody.endsWith("}"))
read[Map[String, Object]](responseBody)
- else if(responseBody.startsWith("[") && responseBody.endsWith("}"))
+ else if (responseBody.startsWith("[") && responseBody.endsWith("}"))
read[List[Map[String, Object]]](responseBody)
- else if(StringUtils.isEmpty(responseBody)) Map.empty[String, Object]
- else if(responseBody.length > 200) throw new HttpException(responseBody.substring(0, 200))
+ else if (StringUtils.isEmpty(responseBody)) Map.empty[String, Object]
+ else if (responseBody.length > 200) throw new HttpException(responseBody.substring(0, 200))
else throw new HttpException(responseBody)
}
override def close(): Unit = {
- discovery.foreach{
+ discovery.foreach {
case d: AbstractDiscovery => IOUtils.closeQuietly(d)
case _ =>
}
- dispatch.Http.shutdown()
+ httpClient.close()
executors.asInstanceOf[ExecutionContextExecutorService].shutdown()
}
-}
-
-class FsPartSource (fs: Fs, path: String) extends PartSource {
- val fsPath = fs.get(path)
- override def getLength: Long = fsPath.getLength
-
- override def createInputStream(): InputStream = fs.read(fsPath)
-
- override def getFileName: String = fsPath.toFile.getName
}
\ No newline at end of file
diff --git a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala
index b2a6e36..4d3d2d4 100644
--- a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala
+++ b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,11 +15,11 @@
import java.util.concurrent.ConcurrentHashMap
-import com.ning.http.client.Response
import com.webank.wedatasphere.linkis.httpclient.Client
import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig
import com.webank.wedatasphere.linkis.httpclient.request.{Action, UserAction}
import org.apache.commons.lang.StringUtils
+import org.apache.http.HttpResponse
/**
* Created by enjoyyin on 2019/5/22.
@@ -76,7 +73,7 @@
protected def getAuthenticationAction(requestAction: Action, serverUrl: String): AuthenticationAction
- def getAuthenticationResult(response: Response, requestAction: AuthenticationAction): AuthenticationResult
+ def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult
def isTimeout(authentication: Authentication): Boolean = System.currentTimeMillis() - authentication.getLastAccessTime >= sessionMaxAliveTime
}
\ No newline at end of file
diff --git a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/HttpAuthentication.scala b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/HttpAuthentication.scala
index 55e6565..e302622 100644
--- a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/HttpAuthentication.scala
+++ b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/authentication/HttpAuthentication.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +17,8 @@
package com.webank.wedatasphere.linkis.httpclient.authentication
-import com.ning.http.client.cookie.Cookie
+import org.apache.http.cookie.Cookie
+
/**
* Created by enjoyyin on 2019/5/16.
@@ -29,6 +27,7 @@
def authToCookies: Array[Cookie]
+
def authToHeaders: java.util.Map[String, String]
}
diff --git a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/discovery/AbstractDiscovery.scala b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/discovery/AbstractDiscovery.scala
index 054b794..7a3d430 100644
--- a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/discovery/AbstractDiscovery.scala
+++ b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/discovery/AbstractDiscovery.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,10 +22,10 @@
import java.util
import java.util.concurrent.ScheduledFuture
-import com.ning.http.client.Response
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.httpclient.Client
import com.webank.wedatasphere.linkis.httpclient.exception.DiscoveryException
+import org.apache.http.HttpResponse
import scala.collection.JavaConversions._
import scala.concurrent.duration.TimeUnit
@@ -129,7 +126,7 @@
protected def getHeartbeatAction(serverUrl: String): HeartbeatAction
- def getHeartbeatResult(response: Response, requestAction: HeartbeatAction): HeartbeatResult
+ def getHeartbeatResult(response: HttpResponse, requestAction: HeartbeatAction): HeartbeatResult
override def addDiscoveryListener(discoveryListener: DiscoveryListener): Unit =
discoveryListeners.add(discoveryListener)
diff --git a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/request/HttpAction.scala b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/request/HttpAction.scala
index 314cf56..9e865fb 100644
--- a/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/request/HttpAction.scala
+++ b/core/httpclient/src/main/scala/com/webank/wedatasphere/linkis/httpclient/request/HttpAction.scala
@@ -1,12 +1,9 @@
/*
* Copyright 2019 WeBank
- *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- *
* http://www.apache.org/licenses/LICENSE-2.0
- *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,7 +19,8 @@
import java.util
-import com.ning.http.client.cookie.Cookie
+import org.apache.http.cookie.Cookie
+
/**
* Created by enjoyyin on 2019/5/16.
@@ -33,12 +31,19 @@
private val cookies = new util.ArrayList[Cookie]
def getHeaders: util.Map[String, String] = headerParams
+
def addHeader(key: String, value: String): Unit = headerParams.put(key, value)
def getCookies: Array[Cookie] = cookies.toArray(new Array[Cookie](cookies.size()))
- def addCookie(cookie: javax.servlet.http.Cookie): Unit =
- cookies.add(Cookie.newValidCookie(cookie.getName, cookie.getValue, cookie.getDomain,
- cookie.getValue, cookie.getPath, -1, cookie.getMaxAge, cookie.getSecure, true))
+
+ /*def addCookie(cookie: javax.servlet.http.Cookie): Unit = {
+ val newCookie: BasicClientCookie = new BasicClientCookie(cookie.getName, cookie.getValue)
+ newCookie.setDomain(cookie.getDomain)
+ newCookie.setPath(cookie.getPath)
+ newCookie.setSecure(true)
+ cookies.add(newCookie)
+ }*/
+
def addCookie(cookie: Cookie): Unit = cookies.add(cookie)
def getURL: String
diff --git a/core/scheduler/pom.xml b/core/scheduler/pom.xml
index 9ec1a9f..487de51 100644
--- a/core/scheduler/pom.xml
+++ b/core/scheduler/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-scheduler</artifactId>
diff --git a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala
index 83e2086..500fa6c 100644
--- a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala
+++ b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala
@@ -16,8 +16,11 @@
package com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue
+import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.scheduler.SchedulerContext
+import com.webank.wedatasphere.linkis.scheduler.event.{ScheduleEvent, SchedulerEventListener}
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorManager
/**
@@ -48,7 +51,7 @@
}
}
- override def getOrCreateExecutorManager = null
+ override def getOrCreateExecutorManager: ExecutorManager = null
- override def getOrCreateSchedulerListenerBus = null
+ override def getOrCreateSchedulerListenerBus: ListenerEventBus[_<: SchedulerEventListener, _<: ScheduleEvent] = null
}
diff --git a/datasource/datasourcemanager/common/pom.xml b/datasource/datasourcemanager/common/pom.xml
new file mode 100644
index 0000000..34841cd
--- /dev/null
+++ b/datasource/datasourcemanager/common/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-datasourcemanager-common</artifactId>
+
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <um.version>1.0.12</um.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/DsmConfiguration.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/DsmConfiguration.java
new file mode 100644
index 0000000..92c22ef
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/DsmConfiguration.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * Configuration
+ * @author kirkzhou
+ * 2020/02/14
+ */
+public class DsmConfiguration {
+
+ public static CommonVars<String> DSM_ADMIN_USER_LIST =
+ CommonVars.apply("wds.linkis.server.dsm.admin.users", "");
+
+
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/ServiceErrorCode.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/ServiceErrorCode.java
new file mode 100644
index 0000000..7647f18
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/ServiceErrorCode.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * Store error code map
+ * @author kirkzhou
+ * 2020/02/11
+ */
+public class ServiceErrorCode {
+
+ public static CommonVars<Integer> TRANSFORM_FORM_ERROR =
+ CommonVars.apply("wds.linkis.server.dsm.error-code.transform", 99987);
+
+ public static CommonVars<Integer> BML_SERVICE_ERROR =
+ CommonVars.apply("wds.linkis.server.dsm.error-code.bml", 99982);
+
+ public static CommonVars<Integer> REMOTE_METADATA_SERVICE_ERROR =
+ CommonVars.apply("wds.linkis.server.dsm.error-code.metadata", 99983);
+
+ public static CommonVars<Integer> PARAM_VALIDATE_ERROR =
+ CommonVars.apply("wds.linkis.server.dsm.error-code.param-validate", 99986);
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSource.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSource.java
new file mode 100644
index 0000000..e35d73a
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSource.java
@@ -0,0 +1,243 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.domain;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.util.json.Json;
+import org.apache.commons.lang.StringUtils;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Size;
+import java.util.*;
+
+/**
+ * Store the data source information
+ * @author kirkzhou
+ * 2020/02/11
+ */
+@JsonSerialize(include= JsonSerialize.Inclusion.NON_EMPTY)
+@JsonIgnoreProperties(value = {"hibernateLazyInitializer", "handler"}, ignoreUnknown = true)
+public class DataSource {
+
+ private Long id;
+
+ /**
+ * Data source name
+ */
+ @NotNull
+ private String dataSourceName;
+
+ /**
+ * Data source description
+ */
+ @Size(min = 0, max = 200)
+ private String dataSourceDesc;
+
+ /**
+ * ID of data source type
+ */
+ @NotNull
+ private Long dataSourceTypeId;
+
+ /**
+ * Identify from creator
+ */
+ private String createIdentify;
+
+ /**
+ * System name from creator
+ */
+ @NotNull
+ private String createSystem;
+ /**
+ * Connection parameters
+ */
+ private Map<String, Object> connectParams = new HashMap<>();
+ /**
+ * Parameter JSON string
+ */
+ @JsonIgnore
+ private String parameter;
+
+ /**
+ * ID of data source environment
+ */
+ private Long dataSourceEnvId;
+
+ /**
+ * Create time
+ */
+ private Date createTime;
+
+ /**
+ * Modify time
+ */
+ private Date modifyTime;
+
+ /**
+ * Modify user
+ */
+ private String modifyUser;
+
+ private String createUser;
+
+ /**
+ * Data source type entity
+ */
+ private DataSourceType dataSourceType;
+
+ /**
+ * Data source env
+ */
+ private DataSourceEnv dataSourceEnv;
+
+ @JsonIgnore
+ private List<DataSourceParamKeyDefinition> keyDefinitions = new ArrayList<>();
+
+ public DataSource(){
+ this.createTime = Calendar.getInstance().getTime();
+ }
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getDataSourceName() {
+ return dataSourceName;
+ }
+
+ public void setDataSourceName(String dataSourceName) {
+ this.dataSourceName = dataSourceName;
+ }
+
+ public String getDataSourceDesc() {
+ return dataSourceDesc;
+ }
+
+ public void setDataSourceDesc(String dataSourceDesc) {
+ this.dataSourceDesc = dataSourceDesc;
+ }
+
+ public Long getDataSourceTypeId() {
+ return dataSourceTypeId;
+ }
+
+ public void setDataSourceTypeId(Long dataSourceTypeId) {
+ this.dataSourceTypeId = dataSourceTypeId;
+ }
+
+ public String getCreateIdentify() {
+ return createIdentify;
+ }
+
+ public void setCreateIdentify(String createIdentify) {
+ this.createIdentify = createIdentify;
+ }
+
+ public String getCreateSystem() {
+ return createSystem;
+ }
+
+ public void setCreateSystem(String createSystem) {
+ this.createSystem = createSystem;
+ }
+
+ public String getParameter() {
+ return parameter;
+ }
+
+ public void setParameter(String parameter) {
+ this.parameter = parameter;
+ }
+
+ public Long getDataSourceEnvId() {
+ return dataSourceEnvId;
+ }
+
+ public void setDataSourceEnvId(Long dataSourceEnvId) {
+ this.dataSourceEnvId = dataSourceEnvId;
+ }
+
+ public Date getCreateTime() {
+ return createTime;
+ }
+
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ public Date getModifyTime() {
+ return modifyTime;
+ }
+
+ public void setModifyTime(Date modifyTime) {
+ this.modifyTime = modifyTime;
+ }
+
+ public Map<String, Object> getConnectParams() {
+ if(connectParams.isEmpty() && StringUtils.isNotBlank(parameter)){
+ connectParams.putAll(Objects.requireNonNull(Json.fromJson(parameter, Map.class)));
+ }
+ return connectParams;
+ }
+
+ public void setConnectParams(Map<String, Object> connectParams) {
+ this.connectParams = connectParams;
+ }
+
+ public String getCreateUser() {
+ return createUser;
+ }
+
+ public void setCreateUser(String createUser) {
+ this.createUser = createUser;
+ }
+
+ public String getModifyUser() {
+ return modifyUser;
+ }
+
+ public void setModifyUser(String modifyUser) {
+ this.modifyUser = modifyUser;
+ }
+
+ public DataSourceType getDataSourceType() {
+ return dataSourceType;
+ }
+
+ public void setDataSourceType(DataSourceType dataSourceType) {
+ this.dataSourceType = dataSourceType;
+ }
+
+ public DataSourceEnv getDataSourceEnv() {
+ return dataSourceEnv;
+ }
+
+ public void setDataSourceEnv(DataSourceEnv dataSourceEnv) {
+ this.dataSourceEnv = dataSourceEnv;
+ }
+
+ public List<DataSourceParamKeyDefinition> getKeyDefinitions() {
+ return keyDefinitions;
+ }
+
+ public void setKeyDefinitions(List<DataSourceParamKeyDefinition> keyDefinitions) {
+ this.keyDefinitions = keyDefinitions;
+ }
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceEnv.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceEnv.java
new file mode 100644
index 0000000..575d246
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceEnv.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.domain;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.util.json.Json;
+import org.apache.commons.lang.StringUtils;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.validation.constraints.NotNull;
+import javax.validation.constraints.Size;
+import java.util.*;
+
+/**
+ * Store the data source environment information
+ * @author kirkzhou
+ * 2020/02/11
+ */
+@JsonSerialize(include= JsonSerialize.Inclusion.NON_EMPTY)
+@JsonIgnoreProperties(value = {"hibernateLazyInitializer", "handler"}, ignoreUnknown = true)
+public class DataSourceEnv {
+
+ private Long id;
+ /**
+ * Environment name
+ */
+ @NotNull
+ private String envName;
+
+ /**
+ * Environment description
+ */
+ @Size(min = 0, max = 200)
+ private String envDesc;
+
+ /**
+ * ID of data source type
+ */
+ @NotNull
+ private Long dataSourceTypeId;
+
+ private DataSourceType dataSourceType;
+ /**
+ * Connection parameters for environment
+ */
+ private Map<String, Object> connectParams = new HashMap<>();
+
+ /**
+ * Parameter JSON string
+ */
+ @JsonIgnore
+ private String parameter;
+
+ /**
+ * Create time
+ */
+ private Date createTime;
+
+ /**
+ * Creator
+ */
+ private String createUser;
+
+ /**
+ * Modify time
+ */
+ private Date modifyTime;
+
+ /**
+ * Modify user
+ */
+ private String modifyUser;
+
+ @JsonIgnore
+ private List<DataSourceParamKeyDefinition> keyDefinitions = new ArrayList<>();
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getEnvName() {
+ return envName;
+ }
+
+ public void setEnvName(String envName) {
+ this.envName = envName;
+ }
+
+ public String getEnvDesc() {
+ return envDesc;
+ }
+
+ public void setEnvDesc(String envDesc) {
+ this.envDesc = envDesc;
+ }
+
+ public Map<String, Object> getConnectParams() {
+ if(connectParams.isEmpty() && StringUtils.isNotBlank(parameter)){
+ connectParams.putAll(Objects.requireNonNull(Json.fromJson(parameter, Map.class)));
+ }
+ return connectParams;
+ }
+
+ public void setConnectParams(Map<String, Object> connectParams) {
+ this.connectParams = connectParams;
+ }
+
+ public String getParameter() {
+ return parameter;
+ }
+
+ public void setParameter(String parameter) {
+ this.parameter = parameter;
+ }
+
+ public Date getCreateTime() {
+ return createTime;
+ }
+
+ public void setCreateTime(Date createTime) {
+ this.createTime = createTime;
+ }
+
+ public String getCreateUser() {
+ return createUser;
+ }
+
+ public void setCreateUser(String createUser) {
+ this.createUser = createUser;
+ }
+
+ public Date getModifyTime() {
+ return modifyTime;
+ }
+
+ public void setModifyTime(Date modifyTime) {
+ this.modifyTime = modifyTime;
+ }
+
+ public String getModifyUser() {
+ return modifyUser;
+ }
+
+ public void setModifyUser(String modifyUser) {
+ this.modifyUser = modifyUser;
+ }
+
+ public Long getDataSourceTypeId() {
+ return dataSourceTypeId;
+ }
+
+ public void setDataSourceTypeId(Long dataSourceTypeId) {
+ this.dataSourceTypeId = dataSourceTypeId;
+ }
+
+ public List<DataSourceParamKeyDefinition> getKeyDefinitions() {
+ return keyDefinitions;
+ }
+
+ public void setKeyDefinitions(List<DataSourceParamKeyDefinition> keyDefinitions) {
+ this.keyDefinitions = keyDefinitions;
+ }
+
+ public DataSourceType getDataSourceType() {
+ return dataSourceType;
+ }
+
+ public void setDataSourceType(DataSourceType dataSourceType) {
+ this.dataSourceType = dataSourceType;
+ }
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceParamKeyDefinition.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceParamKeyDefinition.java
new file mode 100644
index 0000000..6b8360c
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceParamKeyDefinition.java
@@ -0,0 +1,231 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.domain;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.InputStream;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Parameter key definition for data source type
+ * @author kirkzhou
+ * 2020/02/11
+ */
+@JsonSerialize(include= JsonSerialize.Inclusion.NON_EMPTY)
+@JsonIgnoreProperties(value = {"hibernateLazyInitializer", "handler"}, ignoreUnknown = true)
+public class DataSourceParamKeyDefinition {
+ /**
+ * Key-value type
+ */
+ public enum ValueType{
+ /**
+ * Email format
+ */
+ EMAIL(String.class),
+ /**
+ * String
+ */
+ TEXT(String.class),
+ /**
+ * Long
+ */
+ NUMBER(Long.class),
+ /**
+ * SELECT
+ */
+ SELECT(String.class),
+ /**
+ * List
+ */
+ LIST(List.class),
+ /**
+ * Map
+ */
+ MAP(Map.class),
+ /**
+ * RADIO
+ */
+ RADIO(String.class),
+ /**
+ * Password
+ */
+ PASSWORD(String.class),
+ /**
+ * DateTime
+ */
+ DATE(Date.class),
+ /**
+ * File
+ */
+ FILE(InputStream .class);
+ Class<?> type;
+ ValueType(Class<?> type){
+ this.type = type;
+ }
+ public Class<?> getJavaType(){
+ return this.type;
+ }
+ }
+
+ public enum Scope{
+ /**
+ * Env SCOPE
+ */
+ ENV,
+ }
+ /**
+ * Definition id
+ */
+ private Long id;
+
+ /**
+ * Key name
+ */
+ private String key;
+
+ /**
+ * Definition description
+ */
+ private String description;
+
+ /**
+ * Option name
+ */
+ private String name;
+
+ /**
+ * Default value
+ */
+ private String defaultValue;
+
+ /**
+ * Value type
+ */
+ private ValueType valueType;
+ /**
+ * Scope
+ */
+ private Scope scope;
+ /**
+ * If the definition is required
+ */
+ private boolean require;
+ /**
+ * Value regex
+ */
+ private String valueRegex;
+
+ /**
+ * Reference id
+ */
+ private Long refId;
+
+ /**
+ * Reference value
+ */
+ private String refValue;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public void setKey(String key) {
+ this.key = key;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+
+ public void setDefaultValue(String defaultValue) {
+ this.defaultValue = defaultValue;
+ }
+
+ public ValueType getValueType() {
+ return valueType;
+ }
+
+ public void setValueType(ValueType valueType) {
+ this.valueType = valueType;
+ }
+
+ public String getValueRegex() {
+ return valueRegex;
+ }
+
+ public void setValueRegex(String valueRegex) {
+ this.valueRegex = valueRegex;
+ }
+
+ public boolean isRequire() {
+ return require;
+ }
+
+ public void setRequire(boolean require) {
+ this.require = require;
+ }
+
+ public Scope getScope() {
+ return scope;
+ }
+
+ public void setScope(Scope scope) {
+ this.scope = scope;
+ }
+
+ public Long getRefId() {
+ return refId;
+ }
+
+ public void setRefId(Long refId) {
+ this.refId = refId;
+ }
+
+ public String getRefValue() {
+ return refValue;
+ }
+
+ public void setRefValue(String refValue) {
+ this.refValue = refValue;
+ }
+
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceType.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceType.java
new file mode 100644
index 0000000..c273eac
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/domain/DataSourceType.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.domain;
+
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * Data source type entity
+ * @author kirkzhou
+ * 2020/02/11
+ */
+@JsonSerialize(include= JsonSerialize.Inclusion.NON_EMPTY)
+@JsonIgnoreProperties(value = {"hibernateLazyInitializer", "handler"}, ignoreUnknown = true)
+public class DataSourceType {
+
+ private String id;
+ /**
+ * Icon class
+ */
+ private String icon;
+
+ /**
+ * Description
+ */
+ private String description;
+
+ /**
+ * Name
+ */
+ private String name;
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getIcon() {
+ return icon;
+ }
+
+ public void setIcon(String icon) {
+ this.icon = icon;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/CryptoUtils.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/CryptoUtils.java
new file mode 100644
index 0000000..7b2ae4a
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/CryptoUtils.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.util;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.*;
+import java.security.MessageDigest;
+
+/**
+ * @author kirkzhou
+ * 用于密码的加密解密
+ *
+ */
+public class CryptoUtils {
+ private CryptoUtils(){
+ }
+ /**
+ * 序列化对象为String字符
+ *
+ * @param o Object
+ * @return String
+ * @throws Exception
+ */
+ public static String object2String(Serializable o) {
+ try {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(bos);
+ oos.writeObject(o);
+ oos.flush();
+ oos.close();
+ bos.close();
+ return new String(new Base64().encode(bos.toByteArray()));
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * 反序列化字符串为对象
+ *
+ * @param str String
+ * @return Object
+ * @throws IOException
+ * @throws ClassNotFoundException
+ */
+ public static Object string2Object(String str) {
+ try {
+ ByteArrayInputStream bis = new ByteArrayInputStream(new Base64().decode(str.getBytes()));
+ ObjectInputStream ois = new ObjectInputStream(bis);
+ Object o = ois.readObject();
+ bis.close();
+ ois.close();
+ return o;
+ }catch(Exception e){
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ /**
+ * MD5 encrypt
+ * @param source
+ * @param salt
+ * @param iterator
+ * @return
+ */
+ public static String md5(String source, String salt, int iterator){
+ StringBuilder token = new StringBuilder();
+ try{
+ MessageDigest digest = MessageDigest.getInstance("md5");
+ if(StringUtils.isNotEmpty(salt)){
+ digest.update(salt.getBytes("UTF-8"));
+ }
+ byte[] result = digest.digest(source.getBytes());
+ for(int i = 0; i < iterator - 1; i++){
+ digest.reset();
+ result = digest.digest(result);
+ }
+ for (byte aResult : result) {
+ int temp = aResult & 0xFF;
+ if (temp <= 0xF) {
+ token.append("0");
+ }
+ token.append(Integer.toHexString(temp));
+ }
+ }catch(Exception e){
+ throw new RuntimeException(e.getMessage());
+ }
+ return token.toString();
+ }
+
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/PatternInjectUtils.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/PatternInjectUtils.java
new file mode 100644
index 0000000..30d6dbc
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/PatternInjectUtils.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.util;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.util.json.Json;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author kirkzhou
+ * 2018/10/26
+ */
+public class PatternInjectUtils {
+
+ private PatternInjectUtils(){
+
+ }
+
+ private static final String PARAMETER_PREFIX = "[#|$]";
+
+ private static final String ASSIGN_SYMBOL = "=";
+
+ private static final Pattern REGEX = Pattern.compile(
+ "(" + ASSIGN_SYMBOL + "?)" +
+ "("+ PARAMETER_PREFIX + ")" +
+ "\\{([\\w-]+)[|]?([^}]*)}?"
+ );
+
+ /**
+ * inject pattern
+ * @param template
+ * @param params
+ * @param useDefault use default Value
+ * @return
+ */
+ public static String inject(String template, Object[] params, boolean useDefault, boolean escape, boolean placeholder){
+ Matcher matcher = REGEX.matcher(template);
+ StringBuffer sb = new StringBuffer();
+ int offset = 0;
+ while(matcher.find()){
+ String value = "";
+ String extra = "";
+ if( offset < params.length && null != params[offset] ){
+ Object paramsV = params[offset];
+ if( paramsV instanceof String || paramsV instanceof Enum ||
+ paramsV.getClass().isPrimitive() ||
+ isWrapClass(paramsV.getClass())){
+ value = escape? StringEscapeUtils.escapeJava(String.valueOf(paramsV))
+ : String.valueOf(paramsV);
+ }else{
+ value = Json.toJson(paramsV, null);
+ value = escape? StringEscapeUtils.escapeJava(value) : value;
+ }
+ if( null != matcher.group(1)
+ && !"".equals(matcher.group(1))){
+ extra = matcher.group(1);
+ }
+ offset ++;
+ }else if(null != matcher.group(4) && useDefault){
+ value = escape? StringEscapeUtils.escapeJava(String.valueOf(matcher.group(4)))
+ : matcher.group(4);
+ }
+ if(StringUtils.isBlank(value) && !useDefault){
+ value = "\"" + (escape?StringEscapeUtils.escapeJava(matcher.group(3))
+ : matcher.group(3));
+ }else if(!"$".equals(matcher.group(2)) && placeholder){
+ value = "\"" + StringEscapeUtils.escapeJava(value) + "\"";
+ }
+ String result = (extra + value).replace("$", "\\$");
+ matcher.appendReplacement(sb, result);
+ }
+ matcher.appendTail(sb);
+ return sb.toString().replace("\\$","$");
+ }
+ public static String inject(String pattern, Object[] params){
+ return inject(pattern, params, true, true, true);
+ }
+
+ /**
+ * inject pattern
+ * @param template
+ * @param params
+ * @param useDefault
+ * @return
+ */
+ public static String inject(String template, Map<String, Object> params, boolean useDefault, boolean escape, boolean placeholder){
+ Matcher matcher = REGEX.matcher(template);
+ StringBuffer sb = new StringBuffer();
+ //will be more faster?
+ while(matcher.find()){
+ String injected = matcher.group(3);
+ if(null != injected && !"".equals(injected)){
+ int flag = 0;
+ String value = "";
+ String extra = "";
+ for(Map.Entry<String, Object> entry : params.entrySet()){
+ if(injected.equals(entry.getKey()) && null != entry.getValue()){
+ Object entryV = entry.getValue();
+ if(entryV instanceof String || entryV instanceof Enum ||
+ entryV.getClass().isPrimitive() ||
+ isWrapClass(entryV.getClass())){
+ value = escape? StringEscapeUtils.escapeJava(String.valueOf(entryV))
+ : String.valueOf(entryV);
+ }else{
+ value = Json.toJson(entryV, null);
+ value = escape? StringEscapeUtils.escapeJava(value) : value;
+ }
+ if(null != matcher.group(1)
+ || !"".equals(matcher.group(1))){
+ extra = matcher.group(1);
+ }
+ flag = 1;
+ break;
+ }
+ }
+ if(flag == 0 && null != matcher.group(4) && useDefault){
+ value = escape? StringEscapeUtils.escapeJava(String.valueOf(matcher.group(4)))
+ : matcher.group(4);
+ }
+ if(StringUtils.isBlank(value) && !useDefault){
+ value = "\"*#{" + (escape?StringEscapeUtils.escapeJava(matcher.group(3))
+ : matcher.group(3)) + "}*\"";
+ } else if(!"$".equals(matcher.group(2)) && placeholder){
+ value = "\"" + StringEscapeUtils.escapeJava(value) + "\"";
+ }
+ String result = (extra + value).replace("$", "\\$");
+ matcher.appendReplacement(sb, result);
+ }
+ }
+ matcher.appendTail(sb);
+ String print = sb.toString();
+ return print.replace("\\$","$").replace("","");
+ }
+
+ public static String injectPattern(String template, String valuePattern){
+ Matcher matcher = REGEX.matcher(template);
+ StringBuffer sb = new StringBuffer();
+ while(matcher.find()){
+ String extra = matcher.group(1);
+ String value = StringEscapeUtils.escapeJava(matcher.group(3));
+ value = (extra + value.replaceAll("[\\s\\S]+", valuePattern))
+ .replace("$", "\\$");
+ matcher.appendReplacement(sb, value);
+ }
+ matcher.appendTail(sb);
+ return sb.toString().replace("\\$", "$");
+ }
+
+ public static String inject(String template, Map<String, Object> params){
+ return inject(template, params, true, true, true);
+ }
+
+
+ private static boolean isWrapClass(Class clz){
+ try{
+ return ((Class)clz.getField("TYPE").get(null)).isPrimitive();
+ }catch(Exception e){
+ return false;
+ }
+ }
+
+}
diff --git a/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/json/Json.java b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/json/Json.java
new file mode 100644
index 0000000..9becb83
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/common/util/json/Json.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.util.json;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.*;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+/**
+ * @author kirkzhou
+ * Json utils
+ * 2018/9/3
+ */
+public class Json {
+ private static final String PREFIX = "[";
+ private static final String SUFFIX = "]";
+ private static final Logger logger = LoggerFactory.getLogger(Json.class);
+
+ private static ObjectMapper mapper;
+
+ static{
+ mapper = new ObjectMapper();
+ mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
+ mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+ mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
+ mapper.configure(DeserializationFeature.READ_ENUMS_USING_TO_STRING, true);
+ mapper.configure(SerializationFeature.WRITE_ENUMS_USING_TO_STRING, true);
+ mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
+ //empty beans allowed
+ mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+ //ignore unknown properties
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ //cancel to scape non ascii
+ mapper.configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, false);
+ }
+ private Json(){}
+
+ @SuppressWarnings("unchecked")
+ public static <T> T fromJson(String json, Class<?> clazz, Class<?>... parameters){
+ if(StringUtils.isNotBlank(json)){
+ try{
+ if(parameters.length > 0){
+ return (T)mapper.readValue(json, mapper.getTypeFactory().constructParametricType(clazz, parameters));
+ }
+ if(json.startsWith(PREFIX)
+ && json.endsWith(SUFFIX)){
+ JavaType javaType = mapper.getTypeFactory()
+ .constructParametricType(ArrayList.class, clazz);
+ return mapper.readValue(json, javaType);
+ }
+ return (T)mapper.readValue(json, clazz);
+ } catch (Exception e) {
+ logger.info(e.getLocalizedMessage());
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+
+ public static <T> T fromJson(InputStream stream, Class<?> clazz, Class<?>... parameters){
+ StringBuilder builder = new StringBuilder();
+ String jsonStr = null;
+ try{
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
+ while((jsonStr = reader.readLine()) != null){
+ builder.append(jsonStr);
+ }
+ reader.close();
+ }catch(Exception e){
+ logger.info(e.getLocalizedMessage());
+ throw new RuntimeException(e);
+ }
+ return fromJson(builder.toString(), clazz, parameters);
+ }
+
+ public static String toJson(Object obj, Class<?> model){
+ ObjectWriter writer = mapper.writer();
+ if(null != obj){
+ try{
+ if(null != model){
+ writer = writer.withView(model);
+ }
+ return writer.writeValueAsString(obj);
+ } catch (JsonProcessingException e) {
+ logger.info(e.getLocalizedMessage());
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/datasource/datasourcemanager/common/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/common/protocol/DsmQueryProtocol.scala b/datasource/datasourcemanager/common/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/common/protocol/DsmQueryProtocol.scala
new file mode 100644
index 0000000..455e7dd
--- /dev/null
+++ b/datasource/datasourcemanager/common/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/common/protocol/DsmQueryProtocol.scala
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.common.protocol
+
+import java.util
+
+/**
+ * Store error code map
+ * @author kirkzhou
+ * 2020/02/11
+ */
+trait DsmQueryProtocol {
+
+}
+
+/**
+ * Query request of Data Source Information
+ * @param id
+ */
+case class DsInfoQueryRequest(id: String, system: String) extends DsmQueryProtocol
+
+/**
+ * Response of parameter map
+ * @param params
+ */
+case class DsInfoResponse(status: Boolean, dsType: String, params : util.Map[String, Object], creator: String) extends DsmQueryProtocol
+
diff --git a/datasource/datasourcemanager/server/bin/start-dsm-server.sh b/datasource/datasourcemanager/server/bin/start-dsm-server.sh
new file mode 100644
index 0000000..80cc775
--- /dev/null
+++ b/datasource/datasourcemanager/server/bin/start-dsm-server.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+export SERVER_LOG_PATH=$HOME/logs
+export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication
+
+if test -z "$SERVER_HEAP_SIZE"
+then
+ export SERVER_HEAP_SIZE="512M"
+fi
+
+if test -z "$SERVER_JAVA_OPTS"
+then
+ export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log"
+fi
+
+if [[ -f "${SERVER_PID}" ]]; then
+ pid=$(cat ${SERVER_PID})
+ if kill -0 ${pid} >/dev/null 2>&1; then
+ echo "Server is already running."
+ exit 1
+ fi
+fi
+
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+pid=$!
+if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME start failed!"
+ exit 1
+else
+ echo "server $SERVER_NAME start succeeded!"
+ echo $pid > $SERVER_PID
+ sleep 1
+fi
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/bin/stop-dsm-server.sh b/datasource/datasourcemanager/server/bin/stop-dsm-server.sh
new file mode 100644
index 0000000..f032887
--- /dev/null
+++ b/datasource/datasourcemanager/server/bin/stop-dsm-server.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+
+function wait_for_server_to_die() {
+ local pid
+ local count
+ pid=$1
+ timeout=$2
+ count=0
+ timeoutTime=$(date "+%s")
+ let "timeoutTime+=$timeout"
+ currentTime=$(date "+%s")
+ forceKill=1
+
+ while [[ $currentTime -lt $timeoutTime ]]; do
+ $(kill ${pid} > /dev/null 2> /dev/null)
+ if kill -0 ${pid} > /dev/null 2>&1; then
+ sleep 3
+ else
+ forceKill=0
+ break
+ fi
+ currentTime=$(date "+%s")
+ done
+
+ if [[ forceKill -ne 0 ]]; then
+ $(kill -9 ${pid} > /dev/null 2> /dev/null)
+ fi
+}
+
+if [[ ! -f "${SERVER_PID}" ]]; then
+ echo "server $SERVER_NAME is not running"
+else
+ pid=$(cat ${SERVER_PID})
+ if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME is not running"
+ else
+ wait_for_server_to_die $pid 40
+ $(rm -f ${SERVER_PID})
+ echo "server $SERVER_NAME is stopped."
+ fi
+fi
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/conf/application.yml b/datasource/datasourcemanager/server/conf/application.yml
new file mode 100644
index 0000000..eae5f49
--- /dev/null
+++ b/datasource/datasourcemanager/server/conf/application.yml
@@ -0,0 +1,24 @@
+server:
+ port: 8196
+spring:
+ application:
+ name: dsm-server
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://${ip}:${port}/eureka/
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
+
+
+
diff --git a/datasource/datasourcemanager/server/conf/linkis.properties b/datasource/datasourcemanager/server/conf/linkis.properties
new file mode 100644
index 0000000..70b45bd
--- /dev/null
+++ b/datasource/datasourcemanager/server/conf/linkis.properties
@@ -0,0 +1,35 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.server.mybatis.mapperLocations=classpath:com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/*.xml
+wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.linkis.datasourcemanager.common.domain
+wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.linkis.datasourcemanager.core.dao
+
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.datasourcemanager.core.restful
+
+#sit
+wds.linkis.server.version=v1
+
+#test
+wds.linkis.test.mode=true
+wds.linkis.test.user=
+wds.linkis.server.mybatis.datasource.url=
+wds.linkis.server.mybatis.datasource.username=
+wds.linkis.server.mybatis.datasource.password=
+
+#dsm
+wds.linkis.server.dsm.admin.users=
+
+#bml
+wds.linkis.gateway.ip=
+wds.linkis.gateway.port=
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/conf/log4j.properties b/datasource/datasourcemanager/server/conf/log4j.properties
new file mode 100644
index 0000000..d5ee44b
--- /dev/null
+++ b/datasource/datasourcemanager/server/conf/log4j.properties
@@ -0,0 +1,33 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/conf/log4j2.xml b/datasource/datasourcemanager/server/conf/log4j2.xml
new file mode 100644
index 0000000..1c68190
--- /dev/null
+++ b/datasource/datasourcemanager/server/conf/log4j2.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/datasource/datasourcemanager/server/pom.xml b/datasource/datasourcemanager/server/pom.xml
new file mode 100644
index 0000000..d656dee
--- /dev/null
+++ b/datasource/datasourcemanager/server/pom.xml
@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-datasourcemanager-server</artifactId>
+
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- data source manager common dependency-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-datasourcemanager-common</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--jersey bean validation-->
+ <dependency>
+ <groupId>org.glassfish.jersey.ext</groupId>
+ <artifactId>jersey-bean-validation</artifactId>
+ <version>${jersey.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>javax.ws.rs-api</artifactId>
+ <groupId>javax.ws.rs</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hk2-locator</artifactId>
+ <groupId>org.glassfish.hk2</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hk2-api</artifactId>
+ <groupId>org.glassfish.hk2</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--bml client-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-bmlclient</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-mybatis</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <!--rpc-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cloudRPC</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <!--Metadata common-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-metadatamanager-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>linkis-dsm-server</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/java</directory>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>**/*.properties</exclude>
+ <exclude>**/application.yml</exclude>
+ <exclude>**/bootstrap.yml</exclude>
+ <exclude>**/log4j2.xml</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
diff --git a/datasource/datasourcemanager/server/src/main/assembly/distribution.xml b/datasource/datasourcemanager/server/src/main/assembly/distribution.xml
new file mode 100644
index 0000000..4ba803e
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/assembly/distribution.xml
@@ -0,0 +1,206 @@
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-dsm-server</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>linkis-dsm-server</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>true</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+
+ <!-- <excludes>
+ <exclude>antlr:antlr:jar</exclude>
+ <exclude>aopalliance:aopalliance:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude>
+ <exclude>com.google.code.gson:gson:jar</exclude>
+ <exclude>com.google.guava:guava:jar</exclude>
+ <exclude>com.google.protobuf:protobuf-java:jar</exclude>
+ <exclude>com.netflix.archaius:archaius-core:jar</exclude>
+ <exclude>com.netflix.hystrix:hystrix-core:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-core:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude>
+ <exclude>com.netflix.servo:servo-core:jar</exclude>
+ <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude>
+ <exclude>com.sun.jersey:jersey-client:jar</exclude>
+ <exclude>com.sun.jersey:jersey-core:jar</exclude>
+ <exclude>com.sun.jersey:jersey-server:jar</exclude>
+ <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude>
+ <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils-core:jar</exclude>
+ <exclude>commons-cli:commons-cli:jar</exclude>
+ <exclude>commons-collections:commons-collections:jar</exclude>
+ <exclude>commons-configuration:commons-configuration:jar</exclude>
+ <exclude>commons-daemon:commons-daemon:jar</exclude>
+ <exclude>commons-dbcp:commons-dbcp:jar</exclude>
+ <exclude>commons-digester:commons-digester:jar</exclude>
+ <exclude>commons-io:commons-io:jar</exclude>
+ <exclude>commons-lang:commons-lang:jar</exclude>
+ <exclude>commons-net:commons-net:jar</exclude>
+ <exclude>commons-pool:commons-pool:jar</exclude>
+ <exclude>io.netty:netty:jar</exclude>
+ <exclude>io.netty:netty-all:jar</exclude>
+ <exclude>io.netty:netty-buffer:jar</exclude>
+ <exclude>io.netty:netty-codec:jar</exclude>
+ <exclude>io.netty:netty-codec-http:jar</exclude>
+ <exclude>io.netty:netty-common:jar</exclude>
+ <exclude>io.netty:netty-handler:jar</exclude>
+ <exclude>io.netty:netty-transport:jar</exclude>
+ <exclude>io.netty:netty-transport-native-epoll:jar</exclude>
+ <exclude>io.reactivex:rxjava:jar</exclude>
+ <exclude>io.reactivex:rxnetty:jar</exclude>
+ <exclude>io.reactivex:rxnetty-contexts:jar</exclude>
+ <exclude>io.reactivex:rxnetty-servo:jar</exclude>
+ <exclude>javax.activation:activation:jar</exclude>
+ <exclude>javax.annotation:javax.annotation-api:jar</exclude>
+ <exclude>javax.inject:javax.inject:jar</exclude>
+ <exclude>javax.servlet:javax.servlet-api:jar</exclude>
+ <exclude>javax.servlet.jsp:jsp-api:jar</exclude>
+ <exclude>javax.xml.bind:jaxb-api:jar</exclude>
+ <exclude>javax.xml.stream:stax-api:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude>
+ <exclude>org.antlr:antlr-runtime:jar</exclude>
+ <exclude>org.antlr:stringtemplate:jar</exclude>
+ <exclude>org.apache.commons:commons-compress:jar</exclude>
+ <exclude>org.apache.commons:commons-math3:jar</exclude>
+ <exclude>org.apache.curator:curator-framework:jar</exclude>
+ <exclude>org.apache.curator:curator-recipes:jar</exclude>
+ <exclude>org.apache.directory.api:api-asn1-api:jar</exclude>
+ <exclude>org.apache.directory.api:api-util:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-auth:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-common:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude>
+ <exclude>org.apache.htrace:htrace-core:jar</exclude>
+ <exclude>org.apache.logging.log4j:log4j-api:jar</exclude>
+ <exclude>org.apache.zookeeper:zookeeper:jar</exclude>
+ <exclude>org.aspectj:aspectjweaver:jar</exclude>
+ <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude>
+ <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude>
+ <exclude>org.codehaus.jettison:jettison:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-http:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-io:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-plus:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-security:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-server:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-util:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-xml:jar</exclude>
+ <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude>
+ <exclude>org.hdrhistogram:HdrHistogram:jar</exclude>
+ <exclude>org.json4s:json4s-ast_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-core_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-jackson_2.11:jar</exclude>
+ <exclude>org.jsoup:jsoup:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty-util:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude>
+ <exclude>org.scala-lang:scala-compiler:jar</exclude>
+ <exclude>org.scala-lang:scala-library:jar</exclude>
+ <exclude>org.scala-lang:scala-reflect:jar</exclude>
+ <exclude>org.scala-lang:scalap:jar</exclude>
+ <exclude>org.slf4j:jul-to-slf4j:jar</exclude>
+ <exclude>org.slf4j:slf4j-api:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.security:spring-security-crypto:jar</exclude>
+ <exclude>org.springframework.security:spring-security-rsa:jar</exclude>
+ <exclude>org.springframework:spring-aop:jar</exclude>
+ <exclude>org.springframework:spring-beans:jar</exclude>
+ <exclude>org.springframework:spring-context:jar</exclude>
+ <exclude>org.springframework:spring-core:jar</exclude>
+ <exclude>org.springframework:spring-expression:jar</exclude>
+ <!–<exclude>org.springframework:spring-jcl:jar</exclude>–>
+ <exclude>org.springframework:spring-web:jar</exclude>
+ <exclude>org.tukaani:xz:jar</exclude>
+ <exclude>org.yaml:snakeyaml:jar</exclude>
+ <exclude>xerces:xercesImpl:jar</exclude>
+ <exclude>xml-apis:xml-apis:jar</exclude>
+ <exclude>xmlenc:xmlenc:jar</exclude>
+ </excludes>-->
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/conf</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/bin</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>.</directory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <outputDirectory>logs</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceDao.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceDao.java
new file mode 100644
index 0000000..b14fa32
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceDao.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.dao;
+
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceVo;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+/**
+ * Data source dao
+ * @author dennyzhou-zdx
+ * 2020/02/14
+ */
+public interface DataSourceDao {
+
+ /**
+ * Insert
+ * @param dataSource data source
+ */
+ void insertOne(DataSource dataSource);
+
+ /**
+ * View detail
+ * @param dataSourceId data source id
+ * @param createSystem system name
+ * @return data source entity
+ */
+ DataSource selectOneDetail(@Param("dataSourceId") Long dataSourceId,
+ @Param("createSystem") String createSystem);
+
+ /**
+ * View normal
+ * @param dataSourceId data source id
+ * @param createSystem system name
+ * @return data source entity
+ */
+ DataSource selectOne(@Param("dataSourceId") Long dataSourceId,
+ @Param("createSystem") String createSystem);
+ /**
+ * Delete One
+ * @param dataSourceId data source id
+ * @param createSystem create system
+ * @return affect row
+ */
+ int removeOne(@Param("dataSourceId")Long dataSourceId,
+ @Param("createSystem")String createSystem);
+
+
+ /**
+ * Update one
+ * @param updatedOne updated one
+ */
+ void updateOne(DataSource updatedOne);
+
+ /**
+ * Page of query
+ * @param dataSourceVo data source view entity
+ * @return query list
+ */
+ List<DataSource> selectByPageVo(DataSourceVo dataSourceVo);
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java
new file mode 100644
index 0000000..247c419
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.dao;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceEnv;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
+
+import java.util.List;
+
+/**
+ * Data source dao
+ * @author dennyzhou-zdx
+ * 2020/02/14
+ */
+public interface DataSourceEnvDao {
+
+ /**
+ * View details of data source environment information
+ * @param dataSourceEnvId env id
+ * @return
+ */
+ DataSourceEnv selectOneDetail(Long dataSourceEnvId);
+
+ /**
+ * Insert one
+ * @param dataSourceEnv environment
+ */
+ void insertOne(DataSourceEnv dataSourceEnv);
+
+ /**
+ * List all by type id
+ * @param dataSourceTypeId type id
+ * @return
+ */
+ List<DataSourceEnv> listByTypeId(Long dataSourceTypeId);
+
+ /**
+ * Remove one
+ * @param envId env id
+ * @return
+ */
+ int removeOne(Long envId);
+
+ /**
+ * Update one
+ * @param updatedOne
+ */
+ void updateOne(DataSourceEnv updatedOne);
+
+ /**
+ * Page query
+ * @param dataSourceEnvVo environment view entity
+ * @return
+ */
+ List<DataSourceEnv> selectByPageVo(DataSourceEnvVo dataSourceEnvVo);
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceParamKeyDao.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceParamKeyDao.java
new file mode 100644
index 0000000..e6693fb
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceParamKeyDao.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.dao;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+/**
+ *
+ * @author dennyzhou-zdx
+ * 2020/02/14
+ */
+public interface DataSourceParamKeyDao {
+
+ /**
+ * List by data source type id
+ * @param dataSourceTypeId type id
+ * @return
+ */
+ List<DataSourceParamKeyDefinition> listByDataSourceType(Long dataSourceTypeId);
+
+ /**
+ * List by data source type id and scope
+ * @param dataSourceTypeId type id
+ * @param scope scope
+ * @return
+ */
+ List<DataSourceParamKeyDefinition> listByDataSourceTypeAndScope(@Param("dataSourceTypeId")Long dataSourceTypeId,
+ @Param("scope")DataSourceParamKeyDefinition.Scope scope);
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceTypeDao.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceTypeDao.java
new file mode 100644
index 0000000..94a7b6a
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceTypeDao.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.dao;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType;
+
+import java.util.List;
+/**
+ * @author dennyzhou-zdx
+ * 2020/02/14
+ */
+public interface DataSourceTypeDao {
+ /**
+ * Get all types
+ * @return type entity list
+ */
+ List<DataSourceType> getAllTypes();
+
+ /**
+ * View
+ * @param typeId
+ * @return
+ */
+ DataSourceType selectOne(Long typeId);
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceTypeEnvDao.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceTypeEnvDao.java
new file mode 100644
index 0000000..83e6ae4
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/DataSourceTypeEnvDao.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.dao;
+
+import org.apache.ibatis.annotations.Param;
+
+/**
+ * @author dennyzhu-zdx
+ * 2020/02/14
+ */
+public interface DataSourceTypeEnvDao {
+
+ /**
+ * Insert relation between type and environment
+ * @param dataSourceTypeId data source type
+ * @param dataSourceEnvId data source env
+ */
+ void insertRelation(@Param("dataSourceTypeId") Long dataSourceTypeId,
+ @Param("dataSourceEnvId") Long dataSourceEnvId);
+
+ /**
+ * Remove relations by environment id
+ * @param envId
+ * @return
+ */
+ int removeRelationsByEnvId(Long envId);
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml
new file mode 100644
index 0000000..541634b
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceDao">
+ <resultMap id="dataSourceDetailMap" type="DataSource">
+ <result property="id" column="id"/>
+ <result property="dataSourceName" column="datasource_name"/>
+ <result property="dataSourceDesc" column="datasource_desc"/>
+ <result property="dataSourceTypeId" column="datasource_type_id"/>
+ <result property="createIdentify" column="create_identity"/>
+ <result property="createSystem" column="create_system"/>
+ <result property="parameter" column="parameter"/>
+ <result property="dataSourceEnvId" column="datasource_env_id"/>
+ <result property="createTime" column="create_time"/>
+ <result property="modifyTime" column="modify_time"/>
+ <result property="modifyUser" column="modify_user"/>
+ <result property="createUser" column="create_user"/>
+ <association property="dataSourceType" javaType="DataSourceType">
+ <result property="icon" column="icon"/>
+ <result property="name" column="name"/>
+ </association>
+ <association property="dataSourceEnv" javaType="DataSourceEnv" column="datasource_env_id" select="selectEnvById">
+ <result property="envName" column="env_name"/>
+ <result property="parameter" column="env_parameter"/>
+ </association>
+ </resultMap>
+
+ <resultMap id="dataSourceMap" type="DataSource">
+ <result property="id" column="id"/>
+ <result property="dataSourceName" column="datasource_name"/>
+ <result property="dataSourceDesc" column="datasource_desc"/>
+ <result property="dataSourceTypeId" column="datasource_type_id"/>
+ <result property="createIdentify" column="create_identity"/>
+ <result property="createSystem" column="create_system"/>
+ <result property="parameter" column="parameter"/>
+ <result property="dataSourceEnvId" column="datasource_env_id"/>
+ <result property="createTime" column="create_time"/>
+ <result property="modifyTime" column="modify_time"/>
+ <result property="modifyUser" column="modify_user"/>
+ <result property="createUser" column="create_user"/>
+ </resultMap>
+
+ <sql id="data_source_insert_columns">
+ `datasource_name`, `datasource_type_id`, `datasource_desc`,
+ `create_identify`, `create_system`, `create_user`, `parameter`, `create_time`,
+ `modify_user`, `modify_time`, `datasource_env_id`
+ </sql>
+
+ <sql id="data_source_query_columns">
+ `id`, `datasource_name`, `datasource_type_id`, `datasource_desc`,
+ `create_identify`, `create_system`, `create_user`, `parameter`, `create_time`,
+ `modify_user`, `modify_time`, `datasource_env_id`
+ </sql>
+
+ <sql id="data_source_query_page">
+ `id`, `datasource_name`, `datasource_type_id`, `datasource_desc`,
+ `create_identify`, `create_system`, `create_user`, `create_time`,
+ `modify_user`, `modify_time`, `datasource_env_id`
+ </sql>
+ <sql id="data_source_join_type">
+ t.`name`, t.`icon`,
+ d.`id`, d.`datasource_name`, d.`datasource_type_id`, d.`datasource_desc`,
+ d.`create_identify`, d.`create_system`, d.`create_user`, d.`parameter`, d.`create_time`,
+ d.`modify_user`, d.`modify_time`, d.`datasource_env_id`
+ </sql>
+
+ <insert id="insertOne" useGeneratedKeys="true" keyProperty="id" parameterType="DataSource">
+ <![CDATA[INSERT INTO `linkis_datasource`(]]>
+ <include refid="data_source_insert_columns"/>
+ <![CDATA[) VALUES(#{dataSourceName}, #{dataSourceTypeId},
+ #{dataSourceDesc,jdbcType=VARCHAR}, #{createIdentify}, #{createSystem}, #{createUser},
+ #{parameter}, #{createTime}, #{modifyUser,jdbcType=VARCHAR}, #{modifyTime,jdbcType=DATE},
+ #{dataSourceEnvId,jdbcType=INTEGER})]]>
+ </insert>
+
+ <select id="selectOneDetail" resultMap="dataSourceDetailMap">
+ <![CDATA[SELECT ]]>
+ <include refid="data_source_join_type"/>
+ <![CDATA[ FROM `linkis_datasource_type` t
+ INNER JOIN `linkis_datasource` d ON t.`id` = d.`datasource_type_id`]]>
+ <where>
+ d.`id` = #{dataSourceId}
+ <if test="createSystem != null and createSystem != ''">
+ AND d.`create_system` = #{createSystem}
+ </if>
+ </where>
+ </select>
+
+ <select id="selectOne" resultMap="dataSourceMap">
+ <![CDATA[SELECT ]]>
+ <include refid="data_source_query_columns"/>
+ <![CDATA[ FROM `linkis_datasource` ]]>
+ <where>
+ `id` = #{dataSourceId}
+ <if test="createSystem != null and createSystem != ''">
+ AND `create_system` = #{createSystem}
+ </if>
+ </where>
+ </select>
+
+ <select id="selectEnvById" resultType="DataSourceEnv">
+ <![CDATA[SELECT `env_name`, `parameter` AS `env_parameter`
+ FROM `linkis_datasource_env` WHERE `id` = #{id};]]>
+ </select>
+
+ <delete id="removeOne">
+ <![CDATA[DELETE FROM `linkis_datasource`]]>
+ <where>
+ `id` = #{dataSourceId}
+ <if test="createSystem != null and createSystem != ''">
+ AND `create_system` = #{createSystem}
+ </if>
+ </where>
+ </delete>
+
+ <update id="updateOne">
+ <![CDATA[UPDATE `linkis_datasource`]]>
+ <set>
+ <![CDATA[`datasource_name` = #{dataSourceName},]]>
+ <![CDATA[`datasource_desc` = #{dataSourceDesc,jdbcType=VARCHAR},]]>
+ <if test="dataSourceTypeId != null and dataSourceTypeId > 0">
+ <![CDATA[`datasource_type_id` = #{dataSourceTypeId},]]>
+ </if>
+ <![CDATA[`create_identify` = #{createIdentify,jdbcType=VARCHAR},]]>
+ <![CDATA[`create_system` = #{createSystem,jdbcType=VARCHAR},]]>
+ <![CDATA[`parameter` = #{parameter,jdbcType=VARCHAR},]]>
+ <![CDATA[`datasource_env_id` = #{dataSourceEnvId,jdbcType=INTEGER},]]>
+ <![CDATA[`modify_time` = #{modifyTime,jdbcType=DATE},]]>
+ <![CDATA[`modify_user` = #{modifyUser,jdbcType=VARCHAR}]]>
+ </set>
+ <where>
+ `id` = #{id}
+ </where>
+ </update>
+
+ <select id="selectByPageVo" resultMap="dataSourceMap">
+ <![CDATA[SELECT ]]>
+ <include refid="data_source_query_page"/>
+ <![CDATA[ FROM `linkis_datasource` ]]>
+ <where>
+ <![CDATA[`create_system` = #{createSystem}]]>
+ <if test="dataSourceName != null and dataSourceName != ''">
+ <![CDATA[AND `datasource_name` like concat('%', #{dataSourceName}, '%')]]>
+ </if>
+ <if test="dataSourceTypeId != null and dataSourceTypeId > 0">
+ <![CDATA[AND `datasource_type_id` = #{dataSourceTypeId}]]>
+ </if>
+ <if test="createIdentifyList.size() > 0">
+ <choose>
+ <when test="createIdentifyList.size() == 1">
+ <foreach collection="createIdentifyList" separator="" item="item" open="AND " close="">
+ <![CDATA[`create_identity` = #{item}]]>
+ </foreach>
+ </when>
+ <otherwise>
+ <foreach collection="createIdentifyList" separator="," item="item"
+ open="AND `create_identity` IN(" close=")">
+ <![CDATA[#{item}]]>
+ </foreach>
+ </otherwise>
+ </choose>
+ </if>
+ </where>
+ </select>
+</mapper>
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml
new file mode 100644
index 0000000..a2f6eac
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceEnvDao">
+ <resultMap id="dataSourceEnvMap" type="DataSourceEnv">
+ <result property="id" column="id"/>
+ <result property="envName" column="env_name"/>
+ <result property="envDesc" column="env_desc"/>
+ <result property="parameter" column="parameter"/>
+ <result property="createTime" column="create_time"/>
+ <result property="createUser" column="create_user"/>
+ <result property="modifyTime" column="modify_time"/>
+ <result property="modifyUser" column="modify_user"/>
+ <result property="dataSourceTypeId" column="data_source_type_id"/>
+ <association property="dataSourceType" javaType="DataSourceType">
+ <result property="icon" column="icon"/>
+ <result property="name" column="name"/>
+ </association>
+ </resultMap>
+
+ <sql id="data_source_env_insert_columns">
+ `env_name`, `env_desc`, `parameter`, `create_time`,
+ `create_user`, `modify_time`, `modify_user`
+ </sql>
+
+ <sql id="data_source_env_detail">
+ `id`, `env_name`, `env_desc`, `parameter`, `create_time`,
+ `create_user`, `modify_time`, `modify_user`
+ </sql>
+
+ <select id="selectOneDetail" resultMap="dataSourceEnvMap">
+ <![CDATA[SELECT e.`id`, e.`env_name`, e.`env_desc`, e.`parameter`, e.`create_time`,
+ e.`create_user`, e.`modify_time`, e.`modify_user`
+ FROM `linkis_datasource_env` e WHERE id = #{dataSourceEnvId};]]>
+ </select>
+
+ <insert id="insertOne" useGeneratedKeys="true" keyProperty="id" parameterType="DataSourceEnv">
+ <![CDATA[INSERT INTO `linkis_datasource_env`(]]>
+ <include refid="data_source_env_insert_columns"/>
+ <![CDATA[) VALUES(#{envName}, #{envDesc,jdbcType=VARCHAR},
+ #{parameter}, #{createTime}, #{createUser},
+ #{modifyTime,jdbcType=DATE}, #{modifyUser,jdbcType=VARCHAR});]]>
+ </insert>
+
+ <select id="listByTypeId" resultMap="dataSourceEnvMap">
+ <![CDATA[SELECT `id`, `env_name` FROM `linkis_datasource_env`
+ WHERE `id` IN (SELECT `env_id` FROM `linkis_datasource_type_env`
+ WHERE `data_source_type_id` = #{id})]]>
+ </select>
+
+ <select id="selectByPageVo" resultMap="dataSourceEnvMap">
+ <![CDATA[SELECT e.`id`, e.`env_name`, e.`env_desc`, e.`parameter`, e.`create_time`,
+ e.`create_user`, e.`modify_time`, e.`modify_user`, te.`data_source_type_id`
+ FROM `linkis_datasource_env` e INNER JOIN `linkis_datasource_type_env` te
+ ON e.`id` = te.`env_id`]]>
+ <where>
+ <if test="envName != null and envName != ''">
+ <![CDATA[e.`env_name` like concat('%', #{envName}, '%')]]>
+ </if>
+ <if test="dataSourceTypeId != null and dataSourceTypeId > 0">
+ <![CDATA[AND te.`data_source_type_id` = #{dataSourceTypeId}]]>
+ </if>
+ </where>
+ </select>
+
+ <delete id="removeOne">
+ <![CDATA[DELETE FROM `linkis_datasource_env` WHERE `id` = #{id};]]>
+ </delete>
+
+ <update id="updateOne">
+ <![CDATA[UPDATE `linkis_datasource_env` ]]>
+ <set>
+ <![CDATA[`env_name` = #{envName},]]>
+ <![CDATA[`env_desc` = #{envDesc,jdbcType=VARCHAR},]]>
+ <![CDATA[`parameter` = #{parameter},]]>
+ <![CDATA[`parameter` = #{parameter},]]>
+ <![CDATA[`modify_time` = #{modifyTime,jdbcType=DATE},]]>
+ <![CDATA[`modify_user` = #{modifyUser,jdbcType=VARCHAR}]]>
+ </set>
+ <where>
+ `id` = #{id}
+ </where>
+ </update>
+</mapper>
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceParamKeyMapper.xml b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceParamKeyMapper.xml
new file mode 100644
index 0000000..1fcb42d
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceParamKeyMapper.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceParamKeyDao">
+ <resultMap id="paramKeyMap" type="DataSourceParamKeyDefinition">
+ <result property="id" column="id"/>
+ <result property="key" column="key"/>
+ <result property="description" column="description"/>
+ <result property="name" column="name"/>
+ <result property="defaultValue" column="default_value"/>
+ <result property="valueType" column="value_type"/>
+ <result property="scope" column="scope"/>
+ <result property="require" column="require"/>
+ <result property="valueRegex" column="value_regex"/>
+ <result property="refId" column="ref_id"/>
+ <result property="refValue" column="ref_value"/>
+ </resultMap>
+ <sql id="param_key_query">
+ `id`, `key`, `description`, `name`, `require`, `scope`,
+ `default_value`, `value_type`, `value_regex`, `ref_id`, `ref_value`
+ </sql>
+ <select id="listByDataSourceType" resultMap="paramKeyMap" parameterType="Long">
+ <![CDATA[SELECT ]]>
+ <include refid="param_key_query"/>
+ <![CDATA[ FROM `linkis_datasource_type_key` WHERE `data_source_type_id` = #{dataSourceTypeId};]]>
+ </select>
+ <select id="listByDataSourceTypeAndScope" resultMap="paramKeyMap">
+ <![CDATA[SELECT ]]>
+ <include refid="param_key_query"/>
+ <![CDATA[ FROM `linkis_datasource_type_key`
+ WHERE `data_source_type_id` = #{dataSourceTypeId} AND `scope` = #{scope};]]>
+ </select>
+</mapper>
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceTypeEnvMapper.xml b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceTypeEnvMapper.xml
new file mode 100644
index 0000000..3a12401
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceTypeEnvMapper.xml
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceTypeEnvDao">
+
+ <insert id="insertRelation" useGeneratedKeys="true" keyProperty="id">
+ <![CDATA[INSERT INTO `linkis_datasource_type_env`
+ (`data_source_type_id`, `env_id`) VALUES
+ (#{dataSourceTypeId}, #{dataSourceEnvId});]]>
+ </insert>
+
+ <delete id="removeRelationsByEnvId">
+ <![CDATA[DELETE FROM `linkis_datasource_type_env` WHERE
+ `env_id` = #{envId};]]>
+ </delete>
+</mapper>
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceTypeMapper.xml b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceTypeMapper.xml
new file mode 100644
index 0000000..f451e21
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/dao/mapper/DataSourceTypeMapper.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceTypeDao">
+ <resultMap id="dataSourceTypeMap" type="dataSourceType">
+ <result property="id" column="id"/>
+ <result property="icon" column="icon"/>
+ <result property="description" column="description"/>
+ <result property="name" column="name"/>
+ </resultMap>
+
+ <select id="getAllTypes" resultMap="dataSourceTypeMap">
+ <![CDATA[SELECT * FROM `linkis_datasource_type`;]]>
+ </select>
+
+ <select id="selectOne" resultMap="dataSourceTypeMap">
+ <![CDATA[SELECT * FROM `linkis_datasource_type` WHERE id = #{id};]]>
+ </select>
+</mapper>
\ No newline at end of file
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/CustomMultiPartFormDataTransformer.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/CustomMultiPartFormDataTransformer.java
new file mode 100644
index 0000000..8d1b7ed
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/CustomMultiPartFormDataTransformer.java
@@ -0,0 +1,311 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.formdata;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.common.exception.WarnException;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.ServiceErrorCode;
+import org.apache.commons.lang.StringUtils;
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.validation.ConstraintViolation;
+import javax.validation.ConstraintViolationException;
+import javax.validation.ValidationException;
+import javax.validation.Validator;
+import javax.validation.groups.Default;
+import javax.ws.rs.core.MediaType;
+import java.io.ByteArrayInputStream;
+import java.lang.reflect.*;
+import java.util.*;
+
+/**
+ * Custom Transformer of multipart form
+ * @author alexwu
+ * 2020/02/12
+ */
+public class CustomMultiPartFormDataTransformer implements MultiPartFormDataTransformer{
+ private static final Logger LOG = LoggerFactory.getLogger(CustomMultiPartFormDataTransformer.class);
+ @Override
+ public <T> T transformToObject(FormDataMultiPart formData, Class<?> clazz,
+ Validator beanValidator)
+ throws ValidationException, ErrorException {
+ Field[] fields = clazz.getDeclaredFields();
+ Map<String, Field> objectFieldMap = new HashMap<>();
+ Object object;
+ try {
+ object = clazz.getConstructor().newInstance();
+ //Put the bean fields information into map
+ for (Field field : fields) {
+ objectFieldMap.put(field.getName(), field);
+ }
+ Map<String, List<FormDataBodyPart>> formDataParts = formData.getFields();
+ formDataParts.forEach((partName, bodyParts) -> {
+ List<String> stepFieldNames = Arrays.asList(StringUtils.split(partName.replace("[", ".["), "."));
+ String rootFieldName = stepFieldNames.get(0);
+ Field field = objectFieldMap.get(rootFieldName);
+ if (null != field) {
+ try {
+ injectToObject(object, field, stepFieldNames, 0, transformBodyPartToObject(bodyParts));
+ } catch (Exception e) {
+ throw new WarnException(ServiceErrorCode.TRANSFORM_FORM_ERROR.getValue(), "Fail to inject request parameter to object, message:[" + e.getMessage() +"]");
+ }
+ }
+ });
+ }catch(Exception e){
+ LOG.error("Error in transform multipart from: " + e.getMessage());
+ throw new ErrorException(ServiceErrorCode.TRANSFORM_FORM_ERROR.getValue(), "Error in transform multipart from: " + e.getMessage());
+ }
+ if(null != beanValidator){
+ Set<ConstraintViolation<T>> result = beanValidator.validate((T)object, Default.class);
+ if(result.size() > 0){
+ throw new ConstraintViolationException(result);
+ }
+ }
+ return (T)object;
+ }
+
+
+ /**
+ * Transform body parts
+ * @param bodyParts body parts
+ * @return
+ */
+ private Object transformBodyPartToObject(List<FormDataBodyPart> bodyParts){
+ if(bodyParts.size() > 0){
+ Object[] result = new Object[bodyParts.size()];
+ for(int i = 0 ; i < bodyParts.size(); i++){
+ //Get media type
+ FormDataBodyPart bodyPart = bodyParts.get(i);
+ MediaType mediaType = bodyPart.getMediaType();
+ if(mediaType.getType().equals(MediaType.TEXT_PLAIN_TYPE.getType())){
+ result[i] = bodyPart.getValue();
+ }else {
+ //Think others as file
+ FormStreamContent streamContent = new FormStreamContent();
+ FormDataContentDisposition disposition = bodyPart.getFormDataContentDisposition();
+ streamContent.setStream(new ByteArrayInputStream(bodyPart.getValueAs(byte[].class)));
+ streamContent.setFileName(disposition.getFileName());
+ streamContent.setCreateDate(disposition.getCreationDate());
+ streamContent.setModifyDate(disposition.getModificationDate());
+ streamContent.setParameters(disposition.getParameters());
+ result[i] = streamContent;
+ }
+ }
+ return result.length > 1 ? result : result[0];
+ }
+ return null;
+ }
+
+ /**
+ * Inject value to object's field
+ * @param object object injected
+ * @param field field entity
+ * @param stepFieldNames step field name list
+ * @param step step index
+ * @param value actual value
+ * @throws Exception
+ */
+ private void injectToObject(Object object, Field field,
+ List<String> stepFieldNames, int step, Object value) throws Exception {
+ Class<?> fieldType = field.getType();
+ String fieldName = field.getName();
+ if(step + 1 >= stepFieldNames.size()){
+ if(fieldType.equals(String.class) || PrimitiveUtils.isPrimitive(fieldType)){
+ setObjectField(object, field, PrimitiveUtils.primitiveTypeConverse(value, fieldType));
+ } else if (fieldType.equals(Object.class)) {
+ setObjectField(object, field, value);
+ } else {
+ throw new IllegalAccessException("Cannot set value: " + value + " to object field: " + fieldName);
+ }
+ }else if(!PrimitiveUtils.isPrimitive(fieldType)){
+ Object subObject = getObjectField(object, field);
+ if(null == subObject){
+ if(fieldType.equals(Map.class)){
+ subObject = HashMap.class.getConstructor().newInstance();
+ }else if(fieldType.equals(List.class)){
+ subObject = ArrayList.class.getConstructor().newInstance();
+ }else {
+ subObject = fieldType.getConstructor().newInstance();
+ }
+ setObjectField(object, field, subObject);
+ }
+ injectRecurse(subObject, field, stepFieldNames, step + 1, value);
+ }
+
+ }
+
+ /**
+ * Inject value to map structure
+ * @param mapObject map
+ * @param valueType value type
+ * @param stepFieldNames step field name list
+ * @param step step index
+ * @param value actual value
+ * @throws Exception
+ */
+ private void injectToMap(Map mapObject, Class<?> valueType,
+ List<String> stepFieldNames, int step, Object value) throws Exception{
+ String fieldName = stepFieldNames.get(step);
+ if(step + 1 >= stepFieldNames.size()){
+ if(valueType.equals(String.class) || PrimitiveUtils.isPrimitive(valueType)){
+ mapObject.put(fieldName, PrimitiveUtils.primitiveTypeConverse(value, valueType));
+ }else if (valueType.equals(Object.class)){
+ mapObject.put(fieldName, value);
+ }else{
+ throw new IllegalAccessException("Cannot set value: " + value + " to map: " + stepFieldNames.get(step - 1));
+ }
+ }
+ }
+
+ /**
+ * Inject value to list structure
+ * @param listObject list object
+ * @param elementType element type
+ * @param stepFieldNames step field name list
+ * @param step step index
+ * @param value value
+ * @throws Exception
+ */
+ private void injectToList(List listObject, Class<?> elementType,
+ List<String> stepFieldNames, int step, Object value) throws Exception{
+ String fieldName = stepFieldNames.get(step);
+ if(step + 1 >= stepFieldNames.size() && fieldName.matches("\\[\\d+]")){
+ int index = Integer.parseInt(fieldName.substring(1, fieldName.length() - 1));
+ //expand list
+ int expand = index + 1 - listObject.size();
+ while (expand-- > 0) {
+ listObject.add(null);
+ }
+ if(elementType.equals(String.class) || PrimitiveUtils.isPrimitive(elementType)){
+ listObject.set(index, PrimitiveUtils.primitiveTypeConverse(value, elementType));
+ }else if (elementType.equals(Object.class)){
+ listObject.set(index, value);
+ }else{
+ throw new IllegalAccessException("Cannot set value: " + value + " to array: " + stepFieldNames.get(step - 1));
+ }
+ }
+ }
+
+ /**
+ * Inject recursively
+ * @param subObject sub object
+ * @param field sub object's field
+ * @param stepFieldNames step field name list
+ * @param step step index
+ * @param value actual value
+ * @throws Exception
+ */
+ private void injectRecurse(Object subObject, Field field,
+ List<String> stepFieldNames, int step, Object value) throws Exception{
+ Class<?> fieldType = field.getType();
+ if(fieldType.equals(Map.class)){
+ Class<?>[] generic = getGenericTypes(field);
+ if(null == generic || generic[0].equals(String.class)){
+ Class<?> valueType = null == generic ? String.class : generic[1];
+ injectToMap((Map) subObject, valueType, stepFieldNames, step, value);
+ }
+ }else if(fieldType.equals(List.class)){
+ Class<?>[] generic = getGenericTypes(field);
+ injectToList((List)subObject, generic == null ? String.class : generic[0], stepFieldNames,
+ step, value );
+ }else{
+ String nextFieldName = stepFieldNames.get(step);
+ Field nextField = subObject.getClass().getField(nextFieldName);
+ injectToObject(subObject, nextField, stepFieldNames, step, value);
+ }
+ }
+
+ /**
+ * Get generic types
+ * @param field
+ * @return
+ */
+ private Class<?>[] getGenericTypes(Field field){
+ Type fc = field.getGenericType();
+ if(fc instanceof ParameterizedType){
+ Type[] types = ((ParameterizedType)fc).getActualTypeArguments();
+ if(null != types && types.length > 0){
+ Class<?>[] genericClazz = new Class<?>[types.length];
+ for(int i = 0 ; i < genericClazz.length ; i++){
+ genericClazz[i] = (Class<?>)types[i];
+ }
+ return genericClazz;
+ }
+ }
+ return null;
+ }
+
+ private void setObjectField(Object object, Field field, Object value) throws Exception{
+ field.setAccessible(true);
+ field.set(object, value);
+ }
+
+ private Object getObjectField(Object object, Field field) throws Exception{
+ field.setAccessible(true);
+ return field.get(object);
+ }
+
+
+ /**
+ * Tool of primitive
+ */
+ public static class PrimitiveUtils{
+ public static Object primitiveTypeConverse(Object objValue, Class<?> type){
+ if(type.equals(String.class) || null == objValue){
+ return objValue;
+ }
+ String value = String.valueOf(objValue);
+ if(!type.isPrimitive()){
+ try {
+ type = ((Class)type.getField("TYPE").get(null));
+ } catch (Exception e) {
+ //ignore
+ }
+ }
+ switch (type.getSimpleName()) {
+ case "int":
+ return Integer.valueOf(value);
+ case "long":
+ return Long.valueOf(value);
+ case "short":
+ return Short.valueOf(value);
+ case "char":
+ return value.toCharArray()[0];
+ case "float":
+ return Float.valueOf(value);
+ case "double":
+ return Double.valueOf(value);
+ case "byte":
+ return Byte.valueOf(value);
+ case "boolean":
+ return Boolean.valueOf(value);
+ default:
+ throw new RuntimeException("Type: " + type.getSimpleName() + " is not primitive");
+ }
+ }
+
+ public static boolean isPrimitive(Class<?> type){
+ try {
+ return type.isPrimitive() || ((Class) type.getField("TYPE").get(null)).isPrimitive();
+ }catch(Exception e){
+ return false;
+ }
+ }
+ }
+
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/FormDataTransformerFactory.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/FormDataTransformerFactory.java
new file mode 100644
index 0000000..be04770
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/FormDataTransformerFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.formdata;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Simple static factory
+ *
+ */
+public class FormDataTransformerFactory {
+
+ private static final ConcurrentHashMap<String, MultiPartFormDataTransformer>
+ formDataTransformerStored = new ConcurrentHashMap<>();
+
+ /**
+ * Build custom implement
+ * @return
+ */
+ public static MultiPartFormDataTransformer buildCustom(){
+ return formDataTransformerStored.computeIfAbsent("custom", key -> new CustomMultiPartFormDataTransformer());
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/FormStreamContent.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/FormStreamContent.java
new file mode 100644
index 0000000..4f5b318
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/FormStreamContent.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.formdata;
+
+import java.io.InputStream;
+import java.util.Date;
+import java.util.Map;
+
+/**
+ * Stream content
+ * @author alexwu
+ * 2020/02/12
+ */
+public class FormStreamContent {
+ private InputStream stream;
+ private String fileName;
+
+ private long size;
+
+ private Date createDate;
+
+ private Date modifyDate;
+
+ private Map<String, String> parameters;
+
+ public InputStream getStream() {
+ return stream;
+ }
+
+ public void setStream(InputStream stream) {
+ this.stream = stream;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public void setSize(long size) {
+ this.size = size;
+ }
+
+ public Date getCreateDate() {
+ return createDate;
+ }
+
+ public void setCreateDate(Date createDate) {
+ this.createDate = createDate;
+ }
+
+ public Date getModifyDate() {
+ return modifyDate;
+ }
+
+ public void setModifyDate(Date modifyDate) {
+ this.modifyDate = modifyDate;
+ }
+
+ public Map<String, String> getParameters() {
+ return parameters;
+ }
+
+ public void setParameters(Map<String, String> parameters) {
+ this.parameters = parameters;
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/MultiPartFormDataTransformer.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/MultiPartFormDataTransformer.java
new file mode 100644
index 0000000..73156e7
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/formdata/MultiPartFormDataTransformer.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.formdata;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+
+import javax.validation.ValidationException;
+import javax.validation.Validator;
+
+/**
+ * Transformer of multipart form
+ * @author alexwu
+ * 2020/02/12
+ */
+public interface MultiPartFormDataTransformer {
+ /**
+ * Transform the form data to object and validate its fields
+ * @param formData form data
+ * @param clazz clazz
+ * @param beanValidator validator
+ * @param <T>
+ * @return
+ * @throws ValidationException
+ * @throws ErrorException
+ */
+ <T>T transformToObject(FormDataMultiPart formData, Class<?> clazz, Validator beanValidator)
+ throws ValidationException, ErrorException;
+
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java
new file mode 100644
index 0000000..22254d0
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.restful;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceEnv;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.FormDataTransformerFactory;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.MultiPartFormDataTransformer;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceInfoService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceRelateService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidator;
+import com.webank.wedatasphere.linkis.server.Message;
+import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServletRequest;
+import javax.validation.ConstraintViolation;
+import javax.validation.ConstraintViolationException;
+import javax.validation.Validator;
+import javax.validation.groups.Default;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.*;
+
+/**
+ * @author liaoyt
+ * 2020/02/10
+ */
+@Path("/data_source")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+public class DataSourceAdminRestfulApi {
+
+ @Autowired
+ private DataSourceInfoService dataSourceInfoService;
+
+ @Autowired
+ private DataSourceRelateService dataSourceRelateService;
+
+ @Autowired
+ private ParameterValidator parameterValidator;
+
+ @Autowired
+ private Validator beanValidator;
+
+ private MultiPartFormDataTransformer formDataTransformer;
+
+ @PostConstruct
+ public void initRestful(){
+ this.formDataTransformer = FormDataTransformerFactory.buildCustom();
+ }
+
+ @POST
+ @Path("/env/json")
+ public Response insertJsonEnv(DataSourceEnv dataSourceEnv, @Context HttpServletRequest req){
+ return RestfulApiHelper.doAndResponse(()->{
+ String userName = SecurityFilter.getLoginUsername(req);
+ if(!RestfulApiHelper.isAdminUser(userName)){
+ return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
+ }
+ //Bean validation
+ Set<ConstraintViolation<DataSourceEnv>> result = beanValidator.validate(dataSourceEnv, Default.class);
+ if(result.size() > 0){
+ throw new ConstraintViolationException(result);
+ }
+ dataSourceEnv.setCreateUser(userName);
+ insertDataSourceEnv(dataSourceEnv);
+ return Message.ok().data("insert_id", dataSourceEnv.getId());
+ }, "/data_source/env/json","Fail to insert data source environment[新增数据源环境失败]");
+ }
+
+ @POST
+ @Path("/env/form")
+ public Response insertFormEnv(FormDataMultiPart multiPartForm,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(()->{
+ String userName = SecurityFilter.getLoginUsername(request);
+ if(!RestfulApiHelper.isAdminUser(userName)){
+ return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
+ }
+ DataSourceEnv dataSourceEnv = formDataTransformer.transformToObject(multiPartForm, DataSourceEnv.class, beanValidator);
+ dataSourceEnv.setCreateUser(userName);
+ insertDataSourceEnv(dataSourceEnv);
+ return Message.ok().data("insert_id", dataSourceEnv.getId());
+ }, "/data_source/env/form","Fail to insert data source environment[新增数据源环境失败]");
+ }
+
+ @GET
+ @Path("/env_list/all/type/{type_id}")
+ public Response getAllEnvListByDataSourceType(@PathParam("type_id")Long typeId){
+ return RestfulApiHelper.doAndResponse(()->{
+ List<DataSourceEnv> envList = dataSourceInfoService.listDataSourceEnvByType(typeId);
+ return Message.ok().data("env_list", envList);
+ }, "/data_source/env_list/all/type/" + typeId,"Fail to get data source environment list[获取数据源环境清单失败]");
+ }
+
+ @GET
+ @Path("/env/{env_id}")
+ public Response getEnvEntityById(@PathParam("env_id")Long envId){
+ return RestfulApiHelper.doAndResponse(() ->{
+ DataSourceEnv dataSourceEnv = dataSourceInfoService.getDataSourceEnv(envId);
+ return Message.ok().data("env", dataSourceEnv);
+ }, "/data_source/env/" + envId,"Fail to get data source environment[获取数据源环境信息失败]");
+ }
+
+ @DELETE
+ @Path("/env/{env_id}")
+ public Response removeEnvEntity(@PathParam("env_id")Long envId,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(() -> {
+ String userName = SecurityFilter.getLoginUsername(request);
+ if(!RestfulApiHelper.isAdminUser(userName)){
+ return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
+ }
+ Long removeId = dataSourceInfoService.removeDataSourceEnv(envId);
+ if(removeId < 0){
+ return Message.error("Fail to remove data source environment[删除数据源环境信息失败], [id:" +
+ envId + "]");
+ }
+ return Message.ok().data("remove_id", removeId);
+ }, "/data_source/env/" + envId,"Fail to remove data source environment[删除数据源环境信息失败]");
+ }
+
+ @PUT
+ @Path("/env/{env_id}/json")
+ public Response updateJsonEnv(DataSourceEnv dataSourceEnv,
+ @PathParam("env_id")Long envId,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(() -> {
+ String userName = SecurityFilter.getLoginUsername(request);
+ if(!RestfulApiHelper.isAdminUser(userName)){
+ return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
+ }
+ //Bean validation
+ Set<ConstraintViolation<DataSourceEnv>> result = beanValidator.validate(dataSourceEnv, Default.class);
+ if(result.size() > 0){
+ throw new ConstraintViolationException(result);
+ }
+ dataSourceEnv.setId(envId);
+ dataSourceEnv.setModifyUser(userName);
+ dataSourceEnv.setModifyTime(Calendar.getInstance().getTime());
+ DataSourceEnv storedDataSourceEnv = dataSourceInfoService.getDataSourceEnv(envId);
+ if(null == storedDataSourceEnv){
+ return Message.error("Fail to update data source environment[更新数据源环境失败], " + "[Please check the id:'"
+ + envId + " is correct ']");
+ }
+ dataSourceEnv.setCreateUser(storedDataSourceEnv.getCreateUser());
+ updateDataSourceEnv(dataSourceEnv, storedDataSourceEnv);
+ return Message.ok().data("update_id", envId);
+ }, "/data_source/env/" + envId + "/json","Fail to update data source environment[更新数据源环境失败]");
+ }
+
+ @PUT
+ @Path("/env/{env_id}/form")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public Response updateFormEnv(FormDataMultiPart multiPartForm,
+ @PathParam("env_id")Long envId,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(()->{
+ if(null != multiPartForm) {
+ String userName = SecurityFilter.getLoginUsername(request);
+ if (!RestfulApiHelper.isAdminUser(userName)) {
+ return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
+ }
+ DataSourceEnv dataSourceEnv = formDataTransformer.transformToObject(multiPartForm, DataSourceEnv.class, beanValidator);
+ dataSourceEnv.setId(envId);
+ dataSourceEnv.setModifyUser(userName);
+ dataSourceEnv.setModifyTime(Calendar.getInstance().getTime());
+ DataSourceEnv storedDataSourceEnv = dataSourceInfoService.getDataSourceEnv(envId);
+ if (null == storedDataSourceEnv) {
+ return Message.error("Fail to update data source environment[更新数据源环境失败], " + "[Please check the id:'"
+ + envId + " is correct ']");
+ }
+ dataSourceEnv.setCreateUser(storedDataSourceEnv.getCreateUser());
+ updateDataSourceEnv(dataSourceEnv, storedDataSourceEnv);
+ return Message.ok().data("update_id", envId);
+ }
+ return Message.error("Empty request");
+ }, "/data_source/env/" + envId + "/form","Fail to update data source environment[更新数据源环境失败]");
+ }
+
+ @GET
+ @Path("/env")
+ public Response queryDataSourceEnv(@QueryParam("name")String envName,
+ @QueryParam("typeId")Long dataSourceTypeId,
+ @QueryParam("currentPage")Integer currentPage,
+ @QueryParam("pageSize")Integer pageSize){
+ return RestfulApiHelper.doAndResponse(() -> {
+ DataSourceEnvVo dataSourceEnvVo = new DataSourceEnvVo(envName, dataSourceTypeId);
+ dataSourceEnvVo.setCurrentPage(null != currentPage ? currentPage : 1);
+ dataSourceEnvVo.setPageSize(null != pageSize? pageSize : 10);
+ List<DataSourceEnv> queryList = dataSourceInfoService.queryDataSourceEnvPage(dataSourceEnvVo);
+ return Message.ok().data("query_list", queryList);
+ }, "/data_source/env","Fail to query page of data source environment[查询数据源环境失败]");
+ }
+
+ /**
+ * Inner method to insert data source environment
+ * @param dataSourceEnv data source environment entity
+ * @throws ErrorException
+ */
+ private void insertDataSourceEnv(DataSourceEnv dataSourceEnv) throws ErrorException{
+ //Get key definitions in environment scope
+ List<DataSourceParamKeyDefinition> keyDefinitionList = dataSourceRelateService
+ .getKeyDefinitionsByType(dataSourceEnv.getDataSourceTypeId(), DataSourceParamKeyDefinition.Scope.ENV);
+ dataSourceEnv.setKeyDefinitions(keyDefinitionList);
+ Map<String, Object> connectParams = dataSourceEnv.getConnectParams();
+ //Validate connect parameters
+ parameterValidator.validate(keyDefinitionList, connectParams);
+ dataSourceInfoService.saveDataSourceEnv(dataSourceEnv);
+ }
+
+ /**
+ * Inner method to update data source environment
+ * @param updatedOne new entity
+ * @param storedOne old entity
+ * @throws ErrorException
+ */
+ private void updateDataSourceEnv(DataSourceEnv updatedOne, DataSourceEnv storedOne) throws ErrorException{
+ //Get key definitions in environment scope
+ List<DataSourceParamKeyDefinition> keyDefinitionList = dataSourceRelateService
+ .getKeyDefinitionsByType(updatedOne.getDataSourceTypeId(), DataSourceParamKeyDefinition.Scope.ENV);
+ updatedOne.setKeyDefinitions(keyDefinitionList);
+ Map<String, Object> connectParams = updatedOne.getConnectParams();
+ //Validate connect parameters
+ parameterValidator.validate(keyDefinitionList, connectParams);
+ dataSourceInfoService.updateDataSourceEnv(updatedOne, storedOne);
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
new file mode 100644
index 0000000..425cfcc
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.restful;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.FormDataTransformerFactory;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.MultiPartFormDataTransformer;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceInfoService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceRelateService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceVo;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateException;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidator;
+import com.webank.wedatasphere.linkis.server.Message;
+import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
+import org.apache.commons.lang.StringUtils;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServletRequest;
+import javax.validation.ConstraintViolation;
+import javax.validation.ConstraintViolationException;
+import javax.validation.Validator;
+import javax.validation.groups.Default;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.*;
+
+/**
+ * @author liaoyt
+ * 2020/02/10
+ */
+@Path("/data_source")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+@Component
+public class DataSourceCoreRestfulApi {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DataSourceCoreRestfulApi.class);
+ @Autowired
+ private DataSourceInfoService dataSourceInfoService;
+
+ @Autowired
+ private DataSourceRelateService dataSourceRelateService;
+
+ @Autowired
+ private ParameterValidator parameterValidator;
+
+ @Autowired
+ private Validator beanValidator;
+
+ private MultiPartFormDataTransformer formDataTransformer;
+
+
+ @PostConstruct
+ public void initRestful(){
+ this.formDataTransformer = FormDataTransformerFactory.buildCustom();
+ }
+
+ @POST
+ @Path("/info/json")
+ public Response insertJsonInfo(DataSource dataSource, @Context HttpServletRequest req) {
+ return RestfulApiHelper.doAndResponse(() -> {
+ String userName = SecurityFilter.getLoginUsername(req);
+ //Bean validation
+ Set<ConstraintViolation<DataSource>> result = beanValidator.validate(dataSource, Default.class);
+ if(result.size() > 0){
+ throw new ConstraintViolationException(result);
+ }
+ dataSource.setCreateUser(userName);
+ insertDataSourceConfig(dataSource);
+ return Message.ok().data("insert_id", dataSource.getId());
+ }, "/data_source/info/json", "Fail to insert data source[新增数据源失败]");
+ }
+
+ @POST
+ @Path("/info/form")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public Response insertFormConfig(FormDataMultiPart multiPartForm,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(() -> {
+ if(null != multiPartForm) {
+ String userName = SecurityFilter.getLoginUsername(request);
+ DataSource dataSource = formDataTransformer.transformToObject(multiPartForm, DataSource.class, beanValidator);
+ dataSource.setCreateUser(userName);
+ insertDataSourceConfig(dataSource);
+ return Message.ok().data("insert_id", dataSource.getId());
+ }
+ return Message.error("Empty request");
+ }, "/data_source/info/form", "Fail to insert data source[新增数据源失败]");
+ }
+
+ @GET
+ @Path("/info/{data_source_id}")
+ public Response getInfoByDataSourceId(@QueryParam("system")String createSystem,
+ @PathParam("data_source_id")Long dataSourceId,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(() -> {
+ if(StringUtils.isBlank(createSystem)){
+ return Message.error("'create system' is missing[缺少系统名]");
+ }
+ DataSource dataSource = dataSourceInfoService.getDataSourceInfo(dataSourceId, createSystem);
+ //Decrypt
+ if(null != dataSource) {
+ RestfulApiHelper.decryptPasswordKey(dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId())
+ , dataSource.getConnectParams());
+ }
+ return Message.ok().data("info", dataSource);
+ }, "/data_source/info/" + dataSourceId, "Fail to access data source[获取数据源信息失败]");
+ }
+
+
+ @DELETE
+ @Path("/info/{data_source_id}")
+ public Response removeDataSource(@QueryParam("system")String createSystem,
+ @PathParam("data_source_id")Long dataSourceId){
+ return RestfulApiHelper.doAndResponse(() -> {
+ if(StringUtils.isBlank(createSystem)){
+ return Message.error("'create system' is missing[缺少系统名]");
+ }
+ Long removeId = dataSourceInfoService.removeDataSourceInfo(dataSourceId, createSystem);
+ if(removeId < 0){
+ return Message.error("Fail to remove data source[删除数据源信息失败], [id:" + dataSourceId +"]");
+ }
+ return Message.ok().data("remove_id", removeId);
+ }, "/data_source/info/" + dataSourceId,"Fail to remove data source[删除数据源信息失败]");
+ }
+
+ @PUT
+ @Path("/info/{data_source_id}/json")
+ public Response updateDataSourceInJson(DataSource dataSource,
+ @PathParam("data_source_id")Long dataSourceId,
+ @Context HttpServletRequest req){
+ return RestfulApiHelper.doAndResponse(() -> {
+ String userName = SecurityFilter.getLoginUsername(req);
+ //Bean validation
+ Set<ConstraintViolation<DataSource>> result = beanValidator.validate(dataSource, Default.class);
+ if(result.size() > 0){
+ throw new ConstraintViolationException(result);
+ }
+ dataSource.setId(dataSourceId);
+ dataSource.setModifyUser(userName);
+ dataSource.setModifyTime(Calendar.getInstance().getTime());
+ DataSource storedDataSource = dataSourceInfoService.getDataSourceInfoBrief(dataSourceId, dataSource.getCreateSystem());
+ if(null == storedDataSource){
+ return Message.error("Fail to update data source[更新数据源失败], " + "[Please check the id:'"
+ + dataSourceId +"' and create system: '"+dataSource.getCreateSystem()+" is correct ']");
+ }
+ dataSource.setCreateUser(storedDataSource.getCreateUser());
+ updateDataSourceConfig(dataSource, storedDataSource);
+ return Message.ok().data("update_id", dataSourceId);
+ }, "/data_source/info/"+dataSourceId+"/json","Fail to update data source[更新数据源失败]");
+ }
+
+ @PUT
+ @Path("/info/{data_source_id}/form")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public Response updateDataSourceInForm(FormDataMultiPart multiPartForm,
+ @PathParam("data_source_id")Long dataSourceId,
+ @Context HttpServletRequest req){
+ return RestfulApiHelper.doAndResponse(() -> {
+ String userName = SecurityFilter.getLoginUsername(req);
+ DataSource dataSource = formDataTransformer.transformToObject(multiPartForm, DataSource.class, beanValidator);
+ dataSource.setId(dataSourceId);
+ dataSource.setModifyUser(userName);
+ dataSource.setModifyTime(Calendar.getInstance().getTime());
+ DataSource storedDataSource = dataSourceInfoService.getDataSourceInfoBrief(dataSourceId, dataSource.getCreateSystem());
+ if(null == storedDataSource){
+ return Message.error("Fail to update data source[更新数据源失败], " +
+ "[Please check the id:'" + dataSourceId +"' and create system: '"+dataSource.getCreateSystem()+" is correct ']");
+ }
+ dataSource.setCreateUser(storedDataSource.getCreateUser());
+ updateDataSourceConfig(dataSource, storedDataSource);
+ return Message.ok().data("update_id", dataSourceId);
+ }, "/data_source/info/" + dataSourceId + "/form", "Fail to update data source[更新数据源失败]");
+ }
+
+ @GET
+ @Path("/info")
+ public Response queryDataSource(@QueryParam("system")String createSystem,
+ @QueryParam("name")String dataSourceName,
+ @QueryParam("typeId")Long dataSourceTypeId,
+ @QueryParam("identifies")String identifies,
+ @QueryParam("currentPage")Integer currentPage,
+ @QueryParam("pageSize")Integer pageSize){
+ return RestfulApiHelper.doAndResponse(() -> {
+ if(StringUtils.isBlank(createSystem)){
+ return Message.error("'create system' is missing[缺少系统名]");
+ }
+ DataSourceVo dataSourceVo = new DataSourceVo(dataSourceName, dataSourceTypeId,
+ identifies, createSystem);
+ dataSourceVo.setCurrentPage(null != currentPage ? currentPage : 1);
+ dataSourceVo.setPageSize(null != pageSize? pageSize : 10);
+ List<DataSource> queryList = dataSourceInfoService.queryDataSourceInfoPage(dataSourceVo);
+ return Message.ok().data("query_list", queryList);
+ }, "/data_source/info", "Fail to query page of data source[查询数据源失败]");
+ }
+
+ @GET
+ @Path("/key_define/type/{type_id}")
+ public Response getKeyDefinitionsByType(@PathParam("type_id") Long dataSourceTypeId){
+ return RestfulApiHelper.doAndResponse(() -> {
+ List<DataSourceParamKeyDefinition> keyDefinitions = dataSourceRelateService.getKeyDefinitionsByType(dataSourceTypeId);
+ return Message.ok().data("key_define", keyDefinitions);
+ }, "/data_source/key_define/type/" + dataSourceTypeId,
+ "Fail to get key definitions of data source type[查询数据源参数键值对失败]");
+ }
+
+ @GET
+ @Path("/key_define/type/{type_id}/{scope}")
+ public Response getKeyDefinitionsByTypeAndScope(@PathParam("type_id") Long dataSourceTypeId,
+ @PathParam("scope") String scopeValue){
+ return RestfulApiHelper.doAndResponse(() -> {
+ DataSourceParamKeyDefinition.Scope scope = DataSourceParamKeyDefinition.Scope.valueOf(scopeValue.toUpperCase());
+ List<DataSourceParamKeyDefinition> keyDefinitions = dataSourceRelateService
+ .getKeyDefinitionsByType(dataSourceTypeId, scope);
+ return Message.ok().data("key_define", keyDefinitions);
+ }, "/data_source/key_define/type/" + dataSourceTypeId + "/" + scopeValue,
+ "Fail to get key definitions of data source type[查询数据源参数键值对失败]");
+ }
+
+ @GET
+ @Path("/type/all")
+ public Response getAllDataSourceTypes(){
+ return RestfulApiHelper.doAndResponse(() -> {
+ List<DataSourceType> dataSourceTypes = dataSourceRelateService.getAllDataSourceTypes();
+ return Message.ok().data("type_list", dataSourceTypes);
+ }, "/data_source/type/all", "Fail to get all types of data source[获取数据源类型列表失败]");
+ }
+
+ /**
+ * Inner method to insert data source
+ * @param dataSource data source entity
+ * @throws ParameterValidateException
+ */
+ private void insertDataSourceConfig(DataSource dataSource) throws ErrorException {
+ if(null != dataSource.getDataSourceEnvId()){
+ //Merge parameters
+ dataSourceInfoService.addEnvParamsToDataSource(dataSource.getDataSourceEnvId(), dataSource);
+ }
+ //Validate connect parameters
+ List<DataSourceParamKeyDefinition> keyDefinitionList = dataSourceRelateService
+ .getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
+ dataSource.setKeyDefinitions(keyDefinitionList);
+ Map<String,Object> connectParams = dataSource.getConnectParams();
+ parameterValidator.validate(keyDefinitionList, connectParams);
+ //Encrypt password value type
+ RestfulApiHelper.encryptPasswordKey(keyDefinitionList, connectParams);
+ dataSourceInfoService.saveDataSourceInfo(dataSource);
+ }
+
+ /**
+ * Inner method to update data source
+ * @param updatedOne new entity
+ * @param storedOne old entity
+ * @throws ErrorException
+ */
+ private void updateDataSourceConfig(DataSource updatedOne, DataSource storedOne) throws ErrorException{
+ if(null != updatedOne.getDataSourceEnvId()){
+ //Merge parameters
+ dataSourceInfoService.addEnvParamsToDataSource(updatedOne.getDataSourceEnvId(), updatedOne);
+ }
+ //Validate connect parameters
+ List<DataSourceParamKeyDefinition> keyDefinitionList = dataSourceRelateService
+ .getKeyDefinitionsByType(updatedOne.getDataSourceTypeId());
+ updatedOne.setKeyDefinitions(keyDefinitionList);
+ Map<String, Object> connectParams = updatedOne.getConnectParams();
+ parameterValidator.validate(keyDefinitionList, connectParams);
+ RestfulApiHelper.encryptPasswordKey(keyDefinitionList, connectParams);
+ dataSourceInfoService.updateDataSourceInfo(updatedOne, storedOne);
+ }
+
+
+
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceOperateRestfulApi.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceOperateRestfulApi.java
new file mode 100644
index 0000000..4258239
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/DataSourceOperateRestfulApi.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.restful;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.FormDataTransformerFactory;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.MultiPartFormDataTransformer;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceInfoService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceRelateService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.MetadataOperateService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateException;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidator;
+import com.webank.wedatasphere.linkis.metadatamanager.common.MdmConfiguration;
+import com.webank.wedatasphere.linkis.server.Message;
+import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
+import org.apache.commons.lang.StringUtils;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServletRequest;
+import javax.validation.ConstraintViolation;
+import javax.validation.ConstraintViolationException;
+import javax.validation.Validator;
+import javax.validation.groups.Default;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author liaoyt
+ * 2020/02/10
+ */
+@Path("/data_source/op/")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+@Component
+public class DataSourceOperateRestfulApi {
+
+ @Autowired
+ private MetadataOperateService metadataOperateService;
+
+ @Autowired
+ private DataSourceRelateService dataSourceRelateService;
+
+ @Autowired
+ private DataSourceInfoService dataSourceInfoService;
+
+ @Autowired
+ private ParameterValidator parameterValidator;
+
+ @Autowired
+ private Validator beanValidator;
+
+ private MultiPartFormDataTransformer formDataTransformer;
+
+ @PostConstruct
+ public void initRestful(){
+ this.formDataTransformer = FormDataTransformerFactory.buildCustom();
+ }
+
+ @POST
+ @Path("/connect/json")
+ public Response connect(DataSource dataSource,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(() -> {
+ String operator = SecurityFilter.getLoginUsername(request);
+ //Bean validation
+ Set<ConstraintViolation<DataSource>> result = beanValidator.validate(dataSource, Default.class);
+ if(result.size() > 0){
+ throw new ConstraintViolationException(result);
+ }
+ doConnect(operator, dataSource);
+ return Message.ok().data("ok", true);
+ }, "/data_source/op/connect/json","");
+ }
+
+ @POST
+ @Path("/connect/form")
+ public Response connect(FormDataMultiPart multiPartForm,
+ @Context HttpServletRequest request){
+ return RestfulApiHelper.doAndResponse(() -> {
+ String operator = SecurityFilter.getLoginUsername(request);
+ DataSource dataSource = formDataTransformer.transformToObject(multiPartForm, DataSource.class, beanValidator);
+ doConnect(operator, dataSource);
+ return Message.ok().data("ok", true);
+ }, "/data_source/op/connect/form","");
+ }
+
+ /**
+ * Build a connection
+ * @param dataSource
+ */
+ protected void doConnect(String operator, DataSource dataSource) throws ParameterValidateException {
+ if(null != dataSource.getDataSourceEnvId()){
+ dataSourceInfoService.addEnvParamsToDataSource(dataSource.getDataSourceEnvId(), dataSource);
+ }
+ //Validate connect parameters
+ List<DataSourceParamKeyDefinition> keyDefinitionList = dataSourceRelateService
+ .getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
+ dataSource.setKeyDefinitions(keyDefinitionList);
+ Map<String,Object> connectParams = dataSource.getConnectParams();
+ parameterValidator.validate(keyDefinitionList, connectParams);
+ DataSourceType dataSourceType = dataSourceRelateService.getDataSourceType(dataSource.getDataSourceTypeId());
+ metadataOperateService.doRemoteConnect(MdmConfiguration.METADATA_SERVICE_APPLICATION.getValue()
+ + (StringUtils.isNotBlank(dataSourceType.getName())?("-" +dataSourceType.getName().toLowerCase()) : ""),
+ operator, dataSource.getConnectParams());
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/RestfulApiHelper.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/RestfulApiHelper.java
new file mode 100644
index 0000000..0456919
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/RestfulApiHelper.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.restful;
+
+import com.webank.wedatasphere.linkis.common.exception.WarnException;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.DsmConfiguration;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.util.CryptoUtils;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.restful.exception.BeanValidationExceptionMapper;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateException;
+import com.webank.wedatasphere.linkis.server.Message;
+
+import javax.validation.ConstraintViolationException;
+import javax.ws.rs.core.Response;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Helper of restful api entrance
+ * @author liaoyt
+ * 2020/02/10
+ */
+public class RestfulApiHelper {
+ /**
+ * If is administrator
+ * @param userName user name
+ * @return
+ */
+ public static boolean isAdminUser(String userName){
+ List<String> userList = Arrays.asList(DsmConfiguration.DSM_ADMIN_USER_LIST.getValue().split(","));
+ return userList.contains(userName);
+ }
+
+ /**
+ * Encrypt key of password type
+ * @param keyDefinitionList definition list
+ * @param connectParams connection parameters
+ */
+ public static void encryptPasswordKey(List<DataSourceParamKeyDefinition> keyDefinitionList,
+ Map<String, Object> connectParams){
+ keyDefinitionList.forEach(keyDefinition -> {
+ if(keyDefinition.getValueType() == DataSourceParamKeyDefinition.ValueType.PASSWORD){
+ Object password = connectParams.get(keyDefinition.getKey());
+ if(null != password){
+ connectParams.put(keyDefinition.getKey(), CryptoUtils.object2String(String.valueOf(password)));
+ }
+ }
+ });
+ }
+
+ /**
+ * Encrypt key of password type
+ * @param keyDefinitionList definition list
+ * @param connectParams connection parameters
+ */
+ public static void decryptPasswordKey(List<DataSourceParamKeyDefinition> keyDefinitionList,
+ Map<String, Object> connectParams){
+ keyDefinitionList.forEach(keyDefinition -> {
+ if(keyDefinition.getValueType() == DataSourceParamKeyDefinition.ValueType.PASSWORD){
+ Object password = connectParams.get(keyDefinition.getKey());
+ if(null != password){
+ connectParams.put(keyDefinition.getKey(), CryptoUtils.string2Object(String.valueOf(password)));
+ }
+ }
+ });
+ }
+
+ /**
+ *
+ * @param tryOperation operate function
+ * @param failMessage message
+ */
+ public static Response doAndResponse(TryOperation tryOperation, String method, String failMessage){
+ try{
+ Message message = tryOperation.operateAndGetMessage();
+ return Message.messageToResponse(setMethod(message, method));
+ }catch(ParameterValidateException e){
+ return Message.messageToResponse(setMethod(Message.error(e.getMessage()), method));
+ }catch(ConstraintViolationException e){
+ return new BeanValidationExceptionMapper().toResponse(e);
+ }catch(WarnException e){
+ return Message.messageToResponse(setMethod(Message.warn(e.getMessage()), method));
+ }catch(Exception e){
+ return Message.messageToResponse(setMethod(Message.error(failMessage, e), method));
+ }
+ }
+ private static Message setMethod(Message message, String method){
+ message.setMethod(method);
+ return message;
+ }
+
+
+ @FunctionalInterface
+ public interface TryOperation {
+
+ /**
+ * Operate method
+ */
+ Message operateAndGetMessage() throws Exception;
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/exception/BeanValidationExceptionMapper.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/exception/BeanValidationExceptionMapper.java
new file mode 100644
index 0000000..f176dda
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/restful/exception/BeanValidationExceptionMapper.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.restful.exception;
+
+import com.webank.wedatasphere.linkis.server.Message;
+
+import javax.validation.ConstraintViolationException;
+import javax.validation.ValidationException;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+/**
+ * Map bean validation exception to response
+ * @author liaoyt
+ * 2020/02/11
+ */
+@Provider
+public class BeanValidationExceptionMapper implements ExceptionMapper<ValidationException> {
+ @Override
+ public Response toResponse(ValidationException exception) {
+ StringBuilder stringBuilder = new StringBuilder();
+ ((ConstraintViolationException)exception)
+ .getConstraintViolations().forEach(constraintViolation -> stringBuilder.append(constraintViolation.getMessage()).append(";"));
+ Message message = Message.error("Bean validation error[实例校验出错], detail:" + stringBuilder.toString());
+ return Message.messageToResponse(message);
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/BmlAppService.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/BmlAppService.java
new file mode 100644
index 0000000..9725112
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/BmlAppService.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+
+import java.io.InputStream;
+
+/**
+ * BML application service
+ * @author davidhua
+ * 2020/02/14
+ */
+public interface BmlAppService {
+
+ /**
+ * Upload resource by bml client
+ * @param fileName file name
+ * @param inputStream inputStream
+ * @return resource id
+ */
+ String clientUploadResource(String userName, String fileName, InputStream inputStream) throws ErrorException;
+
+ /**
+ * Remove resource by bml client
+ * @param resourceId resource id
+ */
+ void clientRemoveResource(String userName, String resourceId) throws ErrorException;
+
+ /**
+ * Update resource by bml client
+ * @param resourceId resource id
+ * @param inputStream input stream
+ * @return version
+ */
+ String clientUpdateResource(String userName, String resourceId, InputStream inputStream) throws ErrorException;
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceInfoService.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceInfoService.java
new file mode 100644
index 0000000..8592d58
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceInfoService.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceEnv;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceVo;
+
+import java.util.List;
+
+
+/**
+ * @author davidhua
+ * 2020/02/13
+ */
+public interface DataSourceInfoService {
+
+ /**
+ * Save data source information
+ * @param dataSource data source
+ */
+ void saveDataSourceInfo(DataSource dataSource) throws ErrorException;
+
+ /**
+ * Add parameters of data source environment
+ * @param dataSourceEnvId data source environment
+ * @param dataSource data source
+ */
+ void addEnvParamsToDataSource(Long dataSourceEnvId, DataSource dataSource);
+
+ /**
+ * Get data source
+ * @param dataSourceId id
+ * @param createSystem system name
+ * @return data source entity
+ */
+ DataSource getDataSourceInfo(Long dataSourceId, String createSystem);
+
+ /**
+ * Get data source brief information
+ * @param dataSourceId data source id
+ * @param createSystem system
+ * @return
+ */
+ DataSource getDataSourceInfoBrief(Long dataSourceId, String createSystem);
+ /**
+ * Remove data source
+ * @param dataSourceId id
+ * @param createSystem system name
+ * @return
+ */
+ Long removeDataSourceInfo(Long dataSourceId, String createSystem);
+
+ /**
+ * Update data source
+ * @param updatedOne updated data source
+ * @param storedOne stored data source
+ */
+ void updateDataSourceInfo(DataSource updatedOne, DataSource storedOne) throws ErrorException;
+
+ /**
+ * Page query of data source
+ * @param dataSourceVo data source view entity
+ * @return
+ */
+ List<DataSource> queryDataSourceInfoPage(DataSourceVo dataSourceVo);
+
+ /**
+ * Save data source environment
+ * @param dataSourceEnv data source environment
+ */
+ void saveDataSourceEnv(DataSourceEnv dataSourceEnv) throws ErrorException;
+
+ /**
+ * List data source environments
+ * @param dataSourceTypeId type id
+ * @return
+ */
+ List<DataSourceEnv> listDataSourceEnvByType(Long dataSourceTypeId);
+
+ /**
+ * Get data source environment
+ * @param envId environment id
+ * @return
+ */
+ DataSourceEnv getDataSourceEnv(Long envId);
+
+ /**
+ * Remove data source environment
+ * @param envId environment id
+ * @return
+ */
+ Long removeDataSourceEnv(Long envId);
+
+ /**
+ * Update data source environment
+ * @param updatedOne
+ * @param storedOne
+ */
+ void updateDataSourceEnv(DataSourceEnv updatedOne, DataSourceEnv storedOne) throws ErrorException;
+
+ /**
+ * Page query of data source environment
+ * @param dataSourceEnvVo
+ * @return
+ */
+ List<DataSourceEnv> queryDataSourceEnvPage(DataSourceEnvVo dataSourceEnvVo);
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceOpService.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceOpService.java
new file mode 100644
index 0000000..16b0d86
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceOpService.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service;
+
+public interface DataSourceOpService {
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceRelateService.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceRelateService.java
new file mode 100644
index 0000000..165773d
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/DataSourceRelateService.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType;
+
+import java.util.List;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+public interface DataSourceRelateService {
+ /**
+ * Get key definitions by data source type and scope
+ * @param dataSourceTypeId data source type id
+ * @param scope scope
+ * @return
+ */
+ List<DataSourceParamKeyDefinition> getKeyDefinitionsByType(Long dataSourceTypeId,
+ DataSourceParamKeyDefinition.Scope scope);
+
+ /**
+ * Get key definitions by data source type and scope
+ * @param dataSourceTypeId data source type id
+ * @return
+ */
+ List<DataSourceParamKeyDefinition> getKeyDefinitionsByType(Long dataSourceTypeId);
+
+ /**
+ * Get all data source types
+ * @return
+ */
+ List<DataSourceType> getAllDataSourceTypes();
+
+ /**
+ * Get data source type
+ * @param typeId
+ * @return
+ */
+ DataSourceType getDataSourceType(Long typeId);
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/MetadataOperateService.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/MetadataOperateService.java
new file mode 100644
index 0000000..ca46888
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/MetadataOperateService.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.common.exception.WarnException;
+
+import java.util.Map;
+
+/**
+ * Metadata service
+ * @author davidhua
+ * 2020/02/14
+ */
+public interface MetadataOperateService {
+
+ /**
+ * Build connection with parameters in request
+ * @param mdRemoteServiceName metadata remote service
+ * @param operator operate user
+ * @param connectParams parameters
+ * @throws ErrorException
+ */
+ void doRemoteConnect(String mdRemoteServiceName, String operator, Map<String, Object> connectParams) throws WarnException;
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/BmlAppServiceImpl.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/BmlAppServiceImpl.java
new file mode 100644
index 0000000..ad3c26e
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/BmlAppServiceImpl.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service.impl;
+
+import com.webank.wedatasphere.linkis.bml.client.BmlClient;
+import com.webank.wedatasphere.linkis.bml.client.BmlClientFactory;
+import com.webank.wedatasphere.linkis.bml.protocol.BmlDeleteResponse;
+import com.webank.wedatasphere.linkis.bml.protocol.BmlUpdateResponse;
+import com.webank.wedatasphere.linkis.bml.protocol.BmlUploadResponse;
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.ServiceErrorCode;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.BmlAppService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.cloud.context.config.annotation.RefreshScope;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.io.InputStream;
+
+/**
+ * Wrap the communication between Bml service
+ * @author davidhua
+ * 2020/02/15
+ */
+@Service
+@RefreshScope
+public class BmlAppServiceImpl implements BmlAppService {
+ private static final Logger LOG = LoggerFactory.getLogger(BmlAppService.class);
+ /**
+ * Bml client
+ */
+ private BmlClient client;
+
+ @PostConstruct
+ public void buildClient(){
+ client = BmlClientFactory.createBmlClient();
+ }
+ @Override
+ public String clientUploadResource(String userName, String fileName,
+ InputStream inputStream) throws ErrorException{
+ LOG.info("Upload resource to bml server: [ proxy_to_user: " + userName +
+ ", file name:" + fileName + " ]");
+ try{
+ BmlUploadResponse response = client.uploadResource(userName, fileName, inputStream);
+ if(!response.isSuccess()){
+ throw new ErrorException(ServiceErrorCode.BML_SERVICE_ERROR.getValue(), "");
+ }
+ return response.resourceId();
+ }catch(Exception e){
+ LOG.error("Failed to upload resource to bml server[上传资源文件失败], [ proxy_to_user: " + userName +
+ ", file name:" + fileName + " ]", e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void clientRemoveResource(String userName, String resourceId) throws ErrorException{
+ LOG.info("Remove resource to bml server: [ proxy_to_user: " + userName +
+ ", resource id:" + resourceId + " ]");
+ try{
+ BmlDeleteResponse response = client.deleteResource(userName, resourceId);
+ if(!response.isSuccess()){
+ throw new ErrorException(ServiceErrorCode.BML_SERVICE_ERROR.getValue(), "");
+ }
+ }catch(Exception e){
+ LOG.error("Fail to remove resource to bml server[删除资源文件失败], [ proxy_to_user: " + userName +
+ ", resource id:" + resourceId + " ]");
+ throw e;
+ }
+ }
+
+ @Override
+ public String clientUpdateResource(String userName, String resourceId,
+ InputStream inputStream) throws ErrorException{
+ LOG.info("Update resource to bml server: [ proxy_to_user: " + userName +
+ ", resource id:" + resourceId + " ]");
+ try{
+ //File name is invalid;
+ BmlUpdateResponse response = client.updateResource(userName, resourceId, "filename", inputStream);
+ if(!response.isSuccess()){
+ throw new ErrorException(ServiceErrorCode.BML_SERVICE_ERROR.getValue(), "");
+ }
+ return response.version();
+ }catch(Exception e){
+ LOG.error("Fail to update resource to bml server[更新资源文件失败], [ proxy_to_user: " + userName +
+ ", resource id:" + resourceId + " ]");
+ throw e;
+ }
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/DataSourceInfoServiceImpl.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/DataSourceInfoServiceImpl.java
new file mode 100644
index 0000000..c948ed9
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/DataSourceInfoServiceImpl.java
@@ -0,0 +1,370 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service.impl;
+
+import com.github.pagehelper.PageHelper;
+import com.github.pagehelper.PageInfo;
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceParamKeyDao;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceTypeEnvDao;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.FormStreamContent;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.util.json.Json;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceDao;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceEnvDao;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceEnv;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.BmlAppService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceInfoService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.vo.DataSourceVo;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+
+/**
+ * @author davidhua
+ * 2020/02/13
+ */
+@Service
+public class DataSourceInfoServiceImpl implements DataSourceInfoService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DataSourceInfoService.class);
+ @Autowired
+ private BmlAppService bmlAppService;
+
+ @Autowired
+ private DataSourceTypeEnvDao dataSourceTypeEnvDao;
+
+ @Autowired
+ private DataSourceDao dataSourceDao;
+
+ @Autowired
+ private DataSourceEnvDao dataSourceEnvDao;
+
+ @Autowired
+ private DataSourceParamKeyDao dataSourceParamKeyDao;
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void saveDataSourceInfo(DataSource dataSource) throws ErrorException {
+ storeConnectParams(dataSource.getCreateUser(), dataSource.getKeyDefinitions(),
+ dataSource.getConnectParams(), parameter ->{
+ dataSource.setParameter(parameter);
+ //Save information into database
+ dataSourceDao.insertOne(dataSource);
+ });
+ }
+
+ @Override
+ public void addEnvParamsToDataSource(Long dataSourceEnvId, DataSource dataSource) {
+ DataSourceEnv dataSourceEnv = dataSourceEnvDao.selectOneDetail(dataSourceEnvId);
+ if(null != dataSourceEnv){
+ Map<String, Object> envParamMap = dataSourceEnv.getConnectParams();
+ envParamMap.putAll(dataSource.getConnectParams());
+ dataSource.setConnectParams(envParamMap);
+ }
+ }
+
+ @Override
+ public DataSource getDataSourceInfo(Long dataSourceId, String createSystem) {
+ return dataSourceDao.selectOneDetail(dataSourceId, createSystem);
+ }
+
+ @Override
+ public DataSource getDataSourceInfoBrief(Long dataSourceId, String createSystem) {
+ return dataSourceDao.selectOne(dataSourceId, createSystem);
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public Long removeDataSourceInfo(Long dataSourceId, String createSystem) {
+ DataSource dataSource = dataSourceDao.selectOne(dataSourceId, createSystem);
+ if(null != dataSource){
+ //First to delete record in db
+ int affect = dataSourceDao.removeOne(dataSourceId, createSystem);
+ if(affect > 0){
+ //Remove resource
+ Map<String, Object> connectParams = dataSource.getConnectParams();
+ List<DataSourceParamKeyDefinition> keyDefinitions = dataSourceParamKeyDao
+ .listByDataSourceType(dataSource.getDataSourceTypeId());
+ keyDefinitions.forEach(keyDefinition -> {
+ if(keyDefinition.getValueType() == DataSourceParamKeyDefinition.ValueType.FILE
+ && keyDefinition.getScope() != DataSourceParamKeyDefinition.Scope.ENV
+ && connectParams.containsKey(keyDefinition.getKey())){
+ try {
+ //Proxy creator to delete resource
+ bmlAppService.clientRemoveResource(dataSource.getCreateUser(), String
+ .valueOf(connectParams.get(keyDefinition.getKey())));
+ }catch(Exception e){
+ //Ignore remove error
+ }
+ }
+ });
+ return dataSourceId;
+ }
+ }
+ return -1L;
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void updateDataSourceInfo(DataSource updatedOne, DataSource storedOne) throws ErrorException{
+ updateConnectParams(updatedOne.getCreateUser(), updatedOne.getKeyDefinitions(),
+ updatedOne.getConnectParams(), storedOne.getConnectParams(),
+ parameter -> {
+ updatedOne.setParameter(parameter);
+ //Save information into database
+ dataSourceDao.updateOne(updatedOne);
+ });
+ }
+
+ @Override
+ public List<DataSource> queryDataSourceInfoPage(DataSourceVo dataSourceVo) {
+ PageHelper.startPage(dataSourceVo.getCurrentPage(), dataSourceVo.getPageSize());
+ List<DataSource> queryList = dataSourceDao.selectByPageVo(dataSourceVo);
+ PageInfo<DataSource> pageInfo = new PageInfo<>(queryList);
+ return pageInfo.getList();
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void saveDataSourceEnv(DataSourceEnv dataSourceEnv) throws ErrorException {
+ storeConnectParams(dataSourceEnv.getCreateUser(), dataSourceEnv.getKeyDefinitions(),
+ dataSourceEnv.getConnectParams(),
+ parameter ->{
+ dataSourceEnv.setParameter(parameter);
+ //Save environment into database
+ dataSourceEnvDao.insertOne(dataSourceEnv);
+ //Store relation
+ dataSourceTypeEnvDao.insertRelation(dataSourceEnv.getDataSourceTypeId(), dataSourceEnv.getId());
+ });
+ }
+
+ @Override
+ public List<DataSourceEnv> listDataSourceEnvByType(Long dataSourceTypeId) {
+ return dataSourceEnvDao.listByTypeId(dataSourceTypeId);
+ }
+
+ @Override
+ public DataSourceEnv getDataSourceEnv(Long envId) {
+ return dataSourceEnvDao.selectOneDetail(envId);
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public Long removeDataSourceEnv(Long envId) {
+ DataSourceEnv dataSourceEnv = dataSourceEnvDao.selectOneDetail(envId);
+ if(null != dataSourceEnv){
+ //First to delete record in db
+ int affect = dataSourceEnvDao.removeOne(envId);
+ if(affect > 0){
+ //Remove relations
+ dataSourceTypeEnvDao.removeRelationsByEnvId(envId);
+ //Remove resource
+ Map<String, Object> connectParams = dataSourceEnv.getConnectParams();
+ List<DataSourceParamKeyDefinition> keyDefinitions = dataSourceParamKeyDao
+ .listByDataSourceTypeAndScope(dataSourceEnv.getDataSourceTypeId(), DataSourceParamKeyDefinition.Scope.ENV);
+ keyDefinitions.forEach(keyDefinition -> {
+ if(keyDefinition.getValueType() == DataSourceParamKeyDefinition.ValueType.FILE
+ && connectParams.containsKey(keyDefinition.getKey())){
+ try{
+ //Proxy creator to delete resource
+ bmlAppService.clientRemoveResource(dataSourceEnv.getCreateUser(), String
+ .valueOf(connectParams.get(keyDefinition.getKey())));
+ }catch(Exception e){
+ //Ignore remove error
+ }
+ }
+ });
+ return envId;
+ }
+ }
+ return -1L;
+ }
+
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void updateDataSourceEnv(DataSourceEnv updatedOne, DataSourceEnv storedOne) throws ErrorException {
+ updateConnectParams(updatedOne.getCreateUser(), updatedOne.getKeyDefinitions(),
+ updatedOne.getConnectParams(), storedOne.getConnectParams(),
+ parameter ->{
+ updatedOne.setParameter(parameter);
+ //Update environment into database
+ dataSourceEnvDao.updateOne(updatedOne);
+ if(!updatedOne.getDataSourceTypeId().equals(storedOne.getDataSourceTypeId())){
+ //Remove old relation and add new relation
+ dataSourceTypeEnvDao.removeRelationsByEnvId(updatedOne.getId());
+ dataSourceTypeEnvDao.insertRelation(updatedOne.getDataSourceTypeId(), updatedOne.getId());
+ }
+ });
+ }
+
+ @Override
+ public List<DataSourceEnv> queryDataSourceEnvPage(DataSourceEnvVo dataSourceEnvVo) {
+ PageHelper.startPage(dataSourceEnvVo.getCurrentPage(), dataSourceEnvVo.getPageSize());
+ List<DataSourceEnv> queryList = dataSourceEnvDao.selectByPageVo(dataSourceEnvVo);
+ PageInfo<DataSourceEnv> pageInfo = new PageInfo<>(queryList);
+ return pageInfo.getList();
+ }
+
+ /**
+ *
+ * @param userName
+ * @param keyDefinitionList
+ * @param updatedParams
+ * @param storedParams
+ * @param parameterCallback
+ * @throws ErrorException
+ */
+ private void updateConnectParams(String userName, List<DataSourceParamKeyDefinition> keyDefinitionList,
+ Map<String, Object> updatedParams, Map<String, Object> storedParams,
+ Consumer<String> parameterCallback) throws ErrorException {
+ List<String> definedKeyNames = keyDefinitionList.stream().map(DataSourceParamKeyDefinition::getKey)
+ .collect(Collectors.toList());
+ List<String> uploadedResources = new ArrayList<>();
+ try {
+ updatedParams.entrySet().removeIf(entry -> {
+ if (!definedKeyNames.contains(entry.getKey())) {
+ return true;
+ }
+ Object paramValue = entry.getValue();
+ if (paramValue instanceof FormStreamContent) {
+ String resourceId = String.valueOf(storedParams.getOrDefault(entry.getKey(), ""));
+ if (StringUtils.isNotBlank(resourceId)) {
+ uploadFormStream(userName, (FormStreamContent) paramValue, resourceId);
+ } else {
+ resourceId = uploadFormStream(userName, (FormStreamContent) paramValue, "");
+ }
+ if (null == resourceId) {
+ return true;
+ }
+ uploadedResources.add(resourceId);
+ entry.setValue(resourceId);
+ }
+ storedParams.remove(entry.getKey());
+ return false;
+ });
+ //Found the duplicate File
+ List<String> duplicateResources = new ArrayList<>();
+ keyDefinitionList.forEach( definedKey-> {
+ if(definedKey.getValueType() == DataSourceParamKeyDefinition.ValueType.FILE
+ && storedParams.containsKey(definedKey.getKey())){
+ duplicateResources.add(String.valueOf(storedParams.get(definedKey.getKey())));
+ }
+ });
+ parameterCallback.accept(Json.toJson(updatedParams, null));
+ deleteResources(userName, duplicateResources);
+ }catch(Exception e){
+ deleteResources(userName, uploadedResources);
+ if(e.getCause() instanceof ErrorException){
+ throw (ErrorException)e.getCause();
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Upload the form stream context in connect parameters,
+ * and serialize parameters
+ * @param keyDefinitionList
+ * @param connectParams
+ * @param parameterCallback
+ */
+ private void storeConnectParams(String userName, List<DataSourceParamKeyDefinition> keyDefinitionList,
+ Map<String, Object> connectParams,
+ Consumer<String> parameterCallback) throws ErrorException{
+ List<String> definedKeyNames = keyDefinitionList.stream().map(DataSourceParamKeyDefinition::getKey)
+ .collect(Collectors.toList());
+ List<String> uploadedResources = new ArrayList<>();
+ try{
+ connectParams.entrySet().removeIf(entry -> {
+ if(!definedKeyNames.contains(entry.getKey())){
+ return true;
+ }
+ Object paramValue = entry.getValue();
+ //Upload stream resource in connection params
+ if (paramValue instanceof FormStreamContent) {
+ String resourceId = uploadFormStream(userName, (FormStreamContent)paramValue, "");
+ if(null == resourceId) {
+ return true;
+ }
+ uploadedResources.add(resourceId);
+ entry.setValue(resourceId);
+ }
+ return false;
+ });
+ parameterCallback.accept(Json.toJson(connectParams, null));
+ }catch(Exception e){
+ deleteResources(userName, uploadedResources);
+ if(e.getCause() instanceof ErrorException){
+ throw (ErrorException)e.getCause();
+ }
+ throw e;
+ }
+ }
+
+ /**
+ * Upload form stream
+ * @param userName user name
+ * @param streamContent stream content
+ * @param resourceId resource id
+ * @return resource id or version tab
+ */
+ private String uploadFormStream(String userName, FormStreamContent streamContent, String resourceId){
+ String fileName = streamContent.getFileName();
+ InputStream inputStream = streamContent.getStream();
+ if (null != inputStream){
+ //Proxy creator to upload resource
+ try{
+ return StringUtils.isBlank(resourceId)? bmlAppService.clientUploadResource(userName, fileName, inputStream)
+ : bmlAppService.clientUpdateResource(userName, resourceId, inputStream);
+ }catch(Exception e){
+ //Wrap with runtime exception
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Delete uploaded resources
+ * @param userName user name
+ * @param uploadedResources resource id list
+ */
+ private void deleteResources(String userName, List<String> uploadedResources){
+ if(!uploadedResources.isEmpty()){
+ //Remove duplicated resource
+ for (String resourceId : uploadedResources) {
+ try{
+ //Proxy to delete resource
+ bmlAppService.clientRemoveResource(userName, resourceId);
+ }catch(Exception ie){
+ //ignore
+ }
+ }
+ }
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/DataSourceRelateServiceImpl.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/DataSourceRelateServiceImpl.java
new file mode 100644
index 0000000..16c7ad3
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/DataSourceRelateServiceImpl.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service.impl;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceTypeDao;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.dao.DataSourceParamKeyDao;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceType;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.DataSourceRelateService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+@Service
+public class DataSourceRelateServiceImpl implements DataSourceRelateService {
+
+ @Autowired
+ private DataSourceParamKeyDao paramKeyDao;
+
+ @Autowired
+ private DataSourceTypeDao dataSourceTypeDao;
+
+ @Override
+ public List<DataSourceParamKeyDefinition> getKeyDefinitionsByType(Long dataSourceTypeId, DataSourceParamKeyDefinition.Scope scope) {
+ return paramKeyDao.listByDataSourceTypeAndScope(dataSourceTypeId, scope);
+ }
+
+ @Override
+ public List<DataSourceParamKeyDefinition> getKeyDefinitionsByType(Long dataSourceTypeId) {
+ return paramKeyDao.listByDataSourceType(dataSourceTypeId);
+ }
+
+ @Override
+ public List<DataSourceType> getAllDataSourceTypes() {
+ return dataSourceTypeDao.getAllTypes();
+ }
+
+ @Override
+ public DataSourceType getDataSourceType(Long typeId) {
+ return dataSourceTypeDao.selectOne(typeId);
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/MetadataOperateServiceImpl.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/MetadataOperateServiceImpl.java
new file mode 100644
index 0000000..910e26a
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/service/impl/MetadataOperateServiceImpl.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.service.impl;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.common.exception.WarnException;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.FormStreamContent;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.BmlAppService;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.MetadataOperateService;
+import com.webank.wedatasphere.linkis.metadatamanager.common.protocol.MetadataConnect;
+import com.webank.wedatasphere.linkis.metadatamanager.common.protocol.MetadataResponse;
+import com.webank.wedatasphere.linkis.rpc.Sender;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static com.webank.wedatasphere.linkis.datasourcemanager.common.ServiceErrorCode.BML_SERVICE_ERROR;
+import static com.webank.wedatasphere.linkis.datasourcemanager.common.ServiceErrorCode.REMOTE_METADATA_SERVICE_ERROR;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+@Service
+public class MetadataOperateServiceImpl implements MetadataOperateService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetadataOperateService.class);
+ @Autowired
+ private BmlAppService bmlAppService;
+ @Override
+ public void doRemoteConnect(String mdRemoteServiceName ,
+ String operator, Map<String, Object> connectParams) throws WarnException {
+ List<String> uploadedResources = new ArrayList<>();
+ try{
+ connectParams.entrySet().removeIf(entry -> {
+ Object paramValue = entry.getValue();
+ //Upload stream resource in connection parameters
+ if(paramValue instanceof FormStreamContent){
+ FormStreamContent streamContent = (FormStreamContent)paramValue;
+ String fileName = streamContent.getFileName();
+ InputStream inputStream = streamContent.getStream();
+ if (null != inputStream){
+ try {
+ String resourceId = bmlAppService.clientUploadResource(operator,
+ fileName, inputStream);
+ if(null == resourceId){
+ return true;
+ }
+ uploadedResources.add(resourceId);
+ entry.setValue(resourceId);
+ } catch (ErrorException e) {
+ throw new WarnException(BML_SERVICE_ERROR.getValue(), "Fail to operate file in request[上传文件处理失败]");
+ }
+ }
+ }
+ return false;
+ });
+ LOG.info("Send request to metadata service:[" + mdRemoteServiceName + "] for building a connection");
+ //Get a sender
+ Sender sender = Sender.getSender(mdRemoteServiceName);
+ try {
+ Object object = sender.ask(new MetadataConnect(operator, connectParams, ""));
+ if (object instanceof MetadataResponse) {
+ MetadataResponse response = (MetadataResponse) object;
+ if (!response.status()) {
+ throw new WarnException(REMOTE_METADATA_SERVICE_ERROR.getValue(),
+ "Connection Failed[连接失败], Msg[" + response.data() + "]");
+ }
+ } else {
+ throw new WarnException(REMOTE_METADATA_SERVICE_ERROR.getValue(),
+ "Remote Service Error[远端服务出错, 联系运维处理]");
+ }
+ }catch(Throwable t){
+ if(!(t instanceof WarnException)) {
+ throw new WarnException(REMOTE_METADATA_SERVICE_ERROR.getValue(),
+ "Remote Service Error[远端服务出错, 联系运维处理]");
+ }
+ throw t;
+ }
+ }finally{
+ if(!uploadedResources.isEmpty()){
+ uploadedResources.forEach( resourceId ->{
+ try{
+ //Proxy to delete resource
+ bmlAppService.clientRemoveResource(operator, resourceId);
+ }catch(Exception e){
+ //ignore
+ }
+ });
+ }
+ }
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/DataSourceParameterValidator.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/DataSourceParameterValidator.java
new file mode 100644
index 0000000..29e9e72
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/DataSourceParameterValidator.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.validate;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy.RegExpParameterValidateStrategy;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy.TypeParameterValidateStrategy;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author georgeqiao
+ * 2020/02/11
+ */
+
+
+@Component
+public class DataSourceParameterValidator implements ParameterValidator {
+ @PostConstruct
+ public void initToRegister(){
+ registerStrategy(new TypeParameterValidateStrategy());
+ registerStrategy(new RegExpParameterValidateStrategy());
+ }
+ /**
+ * strategies list
+ */
+ private List<ParameterValidateStrategy> strategies = new ArrayList<>();
+
+ @Override
+ public void registerStrategy(ParameterValidateStrategy strategy) {
+ strategies.add(strategy);
+ }
+
+ @Override
+ public void validate(List<DataSourceParamKeyDefinition> paramKeyDefinitions,
+ Map<String, Object> parameters) throws ParameterValidateException{
+ for(DataSourceParamKeyDefinition paramKeyDefinition : paramKeyDefinitions){
+ String keyName = paramKeyDefinition.getKey();
+ Object keyValue = parameters.get(keyName);
+ DataSourceParamKeyDefinition.ValueType valueType = paramKeyDefinition.getValueType();
+ if(null == keyValue){
+ String defaultValue = paramKeyDefinition.getDefaultValue();
+ if(StringUtils.isNotBlank(defaultValue) &&
+ valueType == DataSourceParamKeyDefinition.ValueType.SELECT){
+ defaultValue = defaultValue.split(",")[0].trim();
+ }
+ keyValue = defaultValue;
+ }
+ if(null == keyValue || StringUtils.isBlank(String.valueOf(keyValue))){
+ if(paramKeyDefinition.isRequire()) {
+ throw new ParameterValidateException("Param Validate Failed[参数校验出错], [the value of key: '"
+ + keyName + " cannot be blank']");
+ }
+ continue;
+ }
+ for(ParameterValidateStrategy validateStrategy : strategies){
+ if(validateStrategy.accept(valueType)) {
+ validateStrategy.validate(paramKeyDefinition, keyValue);
+ }
+ }
+ }
+ }
+
+ @Override
+ public List<ParameterValidateStrategy> getStrategies() {
+ return strategies;
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateException.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateException.java
new file mode 100644
index 0000000..448cd73
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateException.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.validate;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+
+import static com.webank.wedatasphere.linkis.datasourcemanager.common.ServiceErrorCode.PARAM_VALIDATE_ERROR;
+
+
+/**
+ * @author georgeqiao
+ * 2020/02/11
+ */
+
+
+public class ParameterValidateException extends ErrorException {
+ public ParameterValidateException(String desc) {
+ super(PARAM_VALIDATE_ERROR.getValue(), desc);
+ }
+
+ public ParameterValidateException(String desc, String ip, int port, String serviceKind) {
+ super(PARAM_VALIDATE_ERROR.getValue(), desc, ip, port, serviceKind);
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateStrategy.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateStrategy.java
new file mode 100644
index 0000000..1d25b30
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidateStrategy.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.validate;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+
+/**
+ * Parameter validate strategy
+ * @author georgeqiao
+ * 2020/02/11
+ */
+public interface ParameterValidateStrategy {
+ /**
+ * If accept value type
+ * @param valueType validate type
+ * @return result
+ */
+ boolean accept(DataSourceParamKeyDefinition.ValueType valueType);
+
+ /**
+ * Validate actual value by key definition
+ * @param keyDefinition key definition
+ * @param actualValue actual value
+ * @return new value
+ */
+ Object validate(DataSourceParamKeyDefinition keyDefinition,
+ Object actualValue) throws ParameterValidateException;
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidator.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidator.java
new file mode 100644
index 0000000..89ff0f7
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/ParameterValidator.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.validate;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author georgeqiao
+ * 2020/02/11
+ */
+
+
+public interface ParameterValidator {
+ /**
+ * Register validate strategy
+ * @param strategy strategy
+ */
+ void registerStrategy(ParameterValidateStrategy strategy);
+
+ /**
+ * Validate parameter dictionary
+ * @param paramKeyDefinitions definitions
+ * @param parameters parameters
+ */
+ void validate(List<DataSourceParamKeyDefinition> paramKeyDefinitions,
+ Map<String, Object> parameters) throws ParameterValidateException;
+
+ /**
+ * Get all strategies
+ * @return
+ */
+ List<ParameterValidateStrategy> getStrategies();
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/RegExpParameterValidateStrategy.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/RegExpParameterValidateStrategy.java
new file mode 100644
index 0000000..64c639a
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/RegExpParameterValidateStrategy.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateException;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateStrategy;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.List;
+
+/**
+ * RegExpression validate strategy
+ * @author georgeqiao
+ * 2020/02/11
+ */
+public class RegExpParameterValidateStrategy implements ParameterValidateStrategy {
+
+
+ @Override
+ public boolean accept(DataSourceParamKeyDefinition.ValueType valueType) {
+ return valueType == DataSourceParamKeyDefinition.ValueType.EMAIL || valueType == DataSourceParamKeyDefinition.ValueType.TEXT || valueType == DataSourceParamKeyDefinition.ValueType.LIST;
+ }
+
+ @Override
+ public Object validate(DataSourceParamKeyDefinition keyDefinition,
+ Object actualValue) throws ParameterValidateException {
+ String valueRegex = keyDefinition.getValueRegex();
+ if(StringUtils.isNotBlank(valueRegex)){
+ if(actualValue instanceof List){
+ List valueList = ((List)actualValue);
+ for(Object value : valueList){
+ match(keyDefinition.getKey(), keyDefinition.getName(), String.valueOf(value), valueRegex);
+ }
+ }else{
+ match(keyDefinition.getKey(), keyDefinition.getName(), String.valueOf(actualValue), valueRegex);
+ }
+
+ }
+ return actualValue;
+ }
+
+ private void match(String key, String name, String value, String valueRegex) throws ParameterValidateException {
+ boolean match = String.valueOf(value).matches(valueRegex);
+ if(!match){
+ throw new ParameterValidateException("Param Validate Failed[参数校验出错], [the value: '"
+ + String.valueOf(value) + "' to key: '"
+ + key + "(" + name +")' doesn't match]");
+ }
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/TypeParameterValidateStrategy.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/TypeParameterValidateStrategy.java
new file mode 100644
index 0000000..bbc0af7
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/validate/strategy/TypeParameterValidateStrategy.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.validate.strategy;
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.FormStreamContent;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateException;
+import com.webank.wedatasphere.linkis.datasourcemanager.core.validate.ParameterValidateStrategy;
+import com.webank.wedatasphere.linkis.metadatamanager.common.Json;
+
+import java.util.List;
+import java.util.Map;
+
+import static com.webank.wedatasphere.linkis.datasourcemanager.core.formdata.CustomMultiPartFormDataTransformer.*;
+
+/**
+ * Type validate strategy
+ * @author georgeqiao
+ * 2020/02/13
+ */
+public class TypeParameterValidateStrategy implements ParameterValidateStrategy {
+ @Override
+ public boolean accept(DataSourceParamKeyDefinition.ValueType valueType) {
+ //Accept all value
+ return true;
+ }
+
+ @Override
+ public Object validate(DataSourceParamKeyDefinition keyDefinition,
+ Object actualValue) throws ParameterValidateException {
+ DataSourceParamKeyDefinition.ValueType valueType = keyDefinition.getValueType();
+ Class<?> javaType = valueType.getJavaType();
+ if(valueType == DataSourceParamKeyDefinition.ValueType.FILE ){
+ if(!actualValue.getClass().equals(FormStreamContent.class)){
+ throw new ParameterValidateException("Param Validate Failed[参数校验出错], [the value of '"
+ + keyDefinition.getKey() + "' must be 'File']");
+ }
+ return actualValue;
+ }
+ if(!javaType.isAssignableFrom(actualValue.getClass())){
+ try {
+ if(javaType.equals(List.class)){
+ return Json.fromJson(String.valueOf(actualValue), List.class, String.class);
+ }else if(javaType.equals(Map.class)){
+ return Json.fromJson(String.valueOf(actualValue), Map.class, String.class, String.class);
+ }else if(PrimitiveUtils.isPrimitive(javaType)){
+ return PrimitiveUtils.primitiveTypeConverse(actualValue, javaType);
+ }
+ }catch(Exception e){
+ throw new ParameterValidateException("Param Validate Failed[参数校验出错], [type of value: '"
+ + actualValue + "' is not '" + javaType.getSimpleName() + "']");
+ }
+ throw new ParameterValidateException("Param Validate Failed[参数校验出错], [type of value: '"
+ + actualValue + "' is not '" + javaType.getSimpleName() + "']");
+ }
+ return actualValue;
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceEnvVo.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceEnvVo.java
new file mode 100644
index 0000000..10a6b80
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceEnvVo.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.vo;
+
+
+/**
+ * @author georgeqiao
+ * 2020/02/11
+ */
+public class DataSourceEnvVo extends PageViewVo{
+
+ private String envName;
+
+ private Long dataSourceTypeId;
+
+ public DataSourceEnvVo(){
+
+ }
+
+ public DataSourceEnvVo(String envName, Long dataSourceTypeId){
+ this.envName = envName;
+ this.dataSourceTypeId = dataSourceTypeId;
+ }
+
+ public String getEnvName() {
+ return envName;
+ }
+
+ public void setEnvName(String envName) {
+ this.envName = envName;
+ }
+
+ public Long getDataSourceTypeId() {
+ return dataSourceTypeId;
+ }
+
+ public void setDataSourceTypeId(Long dataSourceTypeId) {
+ this.dataSourceTypeId = dataSourceTypeId;
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceVo.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceVo.java
new file mode 100644
index 0000000..7a8d2e4
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/DataSourceVo.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.vo;
+
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author georgeqiao
+ * 2020/02/11
+ */
+public class DataSourceVo extends PageViewVo{
+
+ private String dataSourceName;
+
+ private Long dataSourceTypeId;
+
+ private List<String> createIdentifyList = new ArrayList<>();
+
+ private String createSystem;
+
+ public DataSourceVo(){
+
+ }
+
+ public DataSourceVo(String dataSourceName, Long dataSourceTypeId,
+ String createIdentifies, String createSystem){
+ this.dataSourceName = dataSourceName;
+ this.dataSourceTypeId = dataSourceTypeId;
+ if(StringUtils.isNotBlank(createIdentifies)){
+ createIdentifyList.addAll(Arrays.asList(createIdentifies.split(",")));
+ }
+ this.createSystem = createSystem;
+ }
+ public String getDataSourceName() {
+ return dataSourceName;
+ }
+
+ public void setDataSourceName(String dataSourceName) {
+ this.dataSourceName = dataSourceName;
+ }
+
+ public Long getDataSourceTypeId() {
+ return dataSourceTypeId;
+ }
+
+ public void setDataSourceTypeId(Long dataSourceTypeId) {
+ this.dataSourceTypeId = dataSourceTypeId;
+ }
+
+
+ public String getCreateSystem() {
+ return createSystem;
+ }
+
+ public void setCreateSystem(String createSystem) {
+ this.createSystem = createSystem;
+ }
+
+ public List<String> getCreateIdentifyList() {
+ return createIdentifyList;
+ }
+
+ public void setCreateIdentifyList(List<String> createIdentifyList) {
+ this.createIdentifyList = createIdentifyList;
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/PageViewVo.java b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/PageViewVo.java
new file mode 100644
index 0000000..9db3b1f
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/java/com/webank/wedatasphere/linkis/datasourcemanager/core/vo/PageViewVo.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.vo;
+
+/**
+ * @author georgeqiao
+ * 2020/02/11
+ */
+public class PageViewVo {
+ private int currentPage;
+ private int pageSize;
+
+ public int getCurrentPage() {
+ return currentPage;
+ }
+
+ public void setCurrentPage(int currentPage) {
+ this.currentPage = currentPage;
+ }
+
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ public void setPageSize(int pageSize) {
+ this.pageSize = pageSize;
+ }
+}
diff --git a/datasource/datasourcemanager/server/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/core/receivers/DsmReceiver.scala b/datasource/datasourcemanager/server/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/core/receivers/DsmReceiver.scala
new file mode 100644
index 0000000..58f106e
--- /dev/null
+++ b/datasource/datasourcemanager/server/src/main/scala/com/webank/wedatasphere/linkis/datasourcemanager/core/receivers/DsmReceiver.scala
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.datasourcemanager.core.receivers
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.{DataSourceInfoService, DataSourceRelateService}
+import com.webank.wedatasphere.linkis.datasourcemanager.common.protocol.DsInfoResponse
+import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender}
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Component
+
+import scala.concurrent.duration.Duration
+import java.util
+
+import com.webank.wedatasphere.linkis.datasourcemanager.common.domain.DataSource
+import com.webank.wedatasphere.linkis.datasourcemanager.common.protocol.{DsInfoQueryRequest, DsInfoResponse}
+import com.webank.wedatasphere.linkis.datasourcemanager.core.restful.RestfulApiHelper
+import com.webank.wedatasphere.linkis.datasourcemanager.core.service.{DataSourceInfoService, DataSourceRelateService}
+import com.webank.wedatasphere.linkis.datasourcemanager.core.restful.RestfulApiHelper
+/**
+ * @author georgeqiao
+ * 2020/02/10
+ */
+@Component
+class DsmReceiver extends Receiver with Logging{
+
+ @Autowired
+ private var dataSourceInfoService: DataSourceInfoService = _
+
+ @Autowired
+ private var dataSourceRelateService: DataSourceRelateService = _
+
+ override def receive(message: Any, sender: Sender): Unit = ???
+
+ override def receiveAndReply(message: Any, sender: Sender): Any = message match {
+ case DsInfoQueryRequest(id, system) =>
+ if ( id.toLong > 0 && Some(system).isDefined ) {
+ Utils.tryCatch {
+ val dataSource: DataSource = dataSourceInfoService.getDataSourceInfo(id.toLong, system)
+ if ( null != dataSource ) {
+ RestfulApiHelper.decryptPasswordKey(dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId),
+ dataSource.getConnectParams)
+ DsInfoResponse(status = true, dataSource.getDataSourceType.getName,
+ dataSource.getConnectParams, dataSource.getCreateUser)
+ }else DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "")
+ }{
+ case e: Exception => logger.error(s"Fail to get data source information, id:$id system:$system", e)
+ DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "")
+ case t: Throwable => logger.error(s"Fail to get data source information, id:$id system:$system", t)
+ DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "")
+ }
+ } else {
+ DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "")
+ }
+ case _ => new Object()
+ }
+
+ override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = message match {
+ case DsInfoQueryRequest(id, system) =>
+ if ( id.toLong > 0 && Some(system).isDefined ) {
+ Utils.tryCatch {
+ val dataSource: DataSource = dataSourceInfoService.getDataSourceInfo(id.toLong, system)
+ if ( null != dataSource ) {
+ RestfulApiHelper.decryptPasswordKey(dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId),
+ dataSource.getConnectParams)
+ DsInfoResponse(status = true, dataSource.getDataSourceType.getName,
+ dataSource.getConnectParams, dataSource.getCreateUser)
+ }else DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "")
+ }{
+ case e: Exception => logger.error(s"Fail to get data source information, id:$id system:$system", e)
+ DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "")
+ case t: Throwable => logger.error(s"Fail to get data source information, id:$id system:$system", t)
+ DsInfoResponse(status = false, "", new util.HashMap[String, Object](), "")
+ }
+ } else {
+ DsInfoResponse(status = true, "", new util.HashMap[String, Object](), "")
+ }
+ case _ => new Object()
+ }
+}
diff --git a/datasource/metadatamanager/common/pom.xml b/datasource/metadatamanager/common/pom.xml
new file mode 100644
index 0000000..2d3755f
--- /dev/null
+++ b/datasource/metadatamanager/common/pom.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-metadatamanager-common</artifactId>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-common</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--rpc-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cloudRPC</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/Json.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/Json.java
new file mode 100644
index 0000000..203c4a1
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/Json.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.*;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+/**
+ * Json utils
+ * Created by jackyxxie on 2018/9/3.
+ */
+public class Json {
+ private static final String PREFIX = "[";
+ private static final String SUFFIX = "]";
+ private static final Logger logger = LoggerFactory.getLogger(Json.class);
+
+ private static ObjectMapper mapper;
+
+ static{
+ mapper = new ObjectMapper();
+ mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
+ mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+ mapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
+ mapper.configure(DeserializationFeature.READ_ENUMS_USING_TO_STRING, true);
+ mapper.configure(SerializationFeature.WRITE_ENUMS_USING_TO_STRING, true);
+ mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true);
+ //empty beans allowed
+ mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+ //ignore unknown properties
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ //cancel to scape non ascii
+ mapper.configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, false);
+ }
+ private Json(){}
+
+ @SuppressWarnings("unchecked")
+ public static <T> T fromJson(String json, Class<?> clazz, Class<?>... parameters){
+ if(StringUtils.isNotBlank(json)){
+ try{
+ if(parameters.length > 0){
+ return (T)mapper.readValue(json, mapper.getTypeFactory().constructParametricType(clazz, parameters));
+ }
+ if(json.startsWith(PREFIX)
+ && json.endsWith(SUFFIX)){
+ JavaType javaType = mapper.getTypeFactory()
+ .constructParametricType(ArrayList.class, clazz);
+ return mapper.readValue(json, javaType);
+ }
+ return (T)mapper.readValue(json, clazz);
+ } catch (Exception e) {
+ logger.info(e.getLocalizedMessage());
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+
+ public static <T> T fromJson(InputStream stream, Class<?> clazz, Class<?>... parameters){
+ StringBuilder builder = new StringBuilder();
+ String jsonStr = null;
+ try{
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
+ while((jsonStr = reader.readLine()) != null){
+ builder.append(jsonStr);
+ }
+ reader.close();
+ }catch(Exception e){
+ logger.info(e.getLocalizedMessage());
+ throw new RuntimeException(e);
+ }
+ return fromJson(builder.toString(), clazz, parameters);
+ }
+
+ public static String toJson(Object obj, Class<?> model){
+ ObjectWriter writer = mapper.writer();
+ if(null != obj){
+ try{
+ if(null != model){
+ writer = writer.withView(model);
+ }
+ return writer.writeValueAsString(obj);
+ } catch (JsonProcessingException e) {
+ logger.info(e.getLocalizedMessage());
+ throw new RuntimeException(e);
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/MdmConfiguration.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/MdmConfiguration.java
new file mode 100644
index 0000000..480b36d
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/MdmConfiguration.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * Created by jackyxxie on 2020/2/10.
+ */
+public class MdmConfiguration {
+
+ public static CommonVars<String> METADATA_SERVICE_APPLICATION =
+ CommonVars.apply("wds.linkis.server.mdm.service.app.name", "mdm-service");
+
+ public static CommonVars<String> DATA_SOURCE_SERVICE_APPLICATION =
+ CommonVars.apply("wds.linkis.server.dsm.app.name", "dsm-server");
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/CacheConfiguration.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/CacheConfiguration.java
new file mode 100644
index 0000000..01d083d
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/CacheConfiguration.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.cache;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * Created by jackyxxie on 2020/02/14.
+ */
+public class CacheConfiguration {
+
+ public static CommonVars<Long> CACHE_MAX_SIZE =
+ CommonVars.apply("wds.linkis.server.mdm.service.cache.size", 1000L);
+
+ public static CommonVars<Long> CACHE_EXPIRE_TIME =
+ CommonVars.apply("wds.linkis.server.mdm.service.cache.expire", 600L);
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/CacheManager.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/CacheManager.java
new file mode 100644
index 0000000..a1c608a
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/CacheManager.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.cache;
+
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+
+/**
+ * Created by jackyxxie on 2020/2/14.
+ */
+public interface CacheManager {
+ /**
+ * build simple cache
+ * @param cacheId
+ * @param removalListener
+ * @return
+ */
+ <V> Cache<String, V> buildCache(String cacheId, RemovalListener<String, V> removalListener);
+
+ /**
+ * build loading cache
+ * @param cacheId
+ * @param loader
+ * @param removalListener
+ * @return
+ */
+ <V> LoadingCache<String, V> buildCache(String cacheId, CacheLoader<String, V> loader, RemovalListener<String, V> removalListener);
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/ConnCacheManager.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/ConnCacheManager.java
new file mode 100644
index 0000000..7d96236
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/cache/ConnCacheManager.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.cache;
+
+import com.google.common.cache.*;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Created by jackyxxie on 2020/02/14.
+ */
+public class ConnCacheManager implements CacheManager {
+ private ConcurrentHashMap<String, Cache> cacheStore = new ConcurrentHashMap<>();
+ private ConnCacheManager(){
+
+ }
+
+ public static CacheManager custom(){
+ return new ConnCacheManager();
+ }
+
+ @Override
+ public <V> Cache<String, V> buildCache(String cacheId, RemovalListener<String, V> removalListener) {
+ Cache<String, V> vCache = CacheBuilder.newBuilder()
+ .maximumSize(CacheConfiguration.CACHE_MAX_SIZE.getValue())
+ .expireAfterWrite(CacheConfiguration.CACHE_EXPIRE_TIME.getValue(), TimeUnit.SECONDS)
+ .removalListener(removalListener)
+ .build();
+ cacheStore.putIfAbsent(cacheId, vCache);
+ return vCache;
+ }
+
+ @Override
+ public <V> LoadingCache<String, V> buildCache(String cacheId, CacheLoader<String, V> loader,
+ RemovalListener<String, V> removalListener) {
+ LoadingCache<String, V> vCache = CacheBuilder.newBuilder()
+ .maximumSize(CacheConfiguration.CACHE_MAX_SIZE.getValue())
+ .expireAfterWrite(CacheConfiguration.CACHE_EXPIRE_TIME.getValue(), TimeUnit.SECONDS)
+ .removalListener(removalListener)
+ .build(loader);
+ cacheStore.putIfAbsent(cacheId, vCache);
+ return vCache;
+ }
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/domain/MetaColumnInfo.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/domain/MetaColumnInfo.java
new file mode 100644
index 0000000..60911a8
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/domain/MetaColumnInfo.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.domain;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.Serializable;
+
+/**
+ * The meta information of field
+ * Created by jackyxxie on 2020/2/10.
+ */
+@JsonSerialize(include= JsonSerialize.Inclusion.NON_EMPTY)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MetaColumnInfo implements Serializable {
+ private int index = -1;
+ private boolean isPrimaryKey;
+ private String name;
+ private String type;
+
+ public int getIndex() {
+ return index;
+ }
+
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public boolean isPrimaryKey() {
+ return isPrimaryKey;
+ }
+
+ public void setPrimaryKey(boolean primaryKey) {
+ isPrimaryKey = primaryKey;
+ }
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/domain/MetaPartitionInfo.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/domain/MetaPartitionInfo.java
new file mode 100644
index 0000000..d4286c4
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/domain/MetaPartitionInfo.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.domain;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The meta information of partition
+ * Created by jackyxxie on 2020/2/10.
+ */
+@JsonSerialize(include= JsonSerialize.Inclusion.NON_EMPTY)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MetaPartitionInfo implements Serializable {
+ private List<String> partKeys = new ArrayList<>();
+
+ private String name;
+ /**
+ * Partition tree
+ */
+ private PartitionNode root;
+
+ @JsonSerialize(include= JsonSerialize.Inclusion.NON_EMPTY)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class PartitionNode{
+ /**
+ * Node name
+ */
+ private String name;
+ /**
+ * Key: partition value
+ * Value: child partition node
+ */
+ private Map<String, PartitionNode> partitions = new HashMap<>();
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Map<String, PartitionNode> getPartitions() {
+ return partitions;
+ }
+
+ public void setPartitions(Map<String, PartitionNode> partitions) {
+ this.partitions = partitions;
+ }
+ }
+
+ public List<String> getPartKeys() {
+ return partKeys;
+ }
+
+ public void setPartKeys(List<String> partKeys) {
+ this.partKeys = partKeys;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public PartitionNode getRoot() {
+ return root;
+ }
+
+ public void setRoot(PartitionNode root) {
+ this.root = root;
+ }
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/exception/MetaRuntimeException.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/exception/MetaRuntimeException.java
new file mode 100644
index 0000000..e3dcb0e
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/exception/MetaRuntimeException.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.exception;
+
+import com.webank.wedatasphere.linkis.common.exception.WarnException;
+
+/**
+ * Created by jackyxxie on 2020/2/10.
+ */
+public class MetaRuntimeException extends WarnException {
+ private static final int ERROR_CODE = 99900;
+ public MetaRuntimeException(String desc) {
+ super(ERROR_CODE, desc);
+ }
+
+ public MetaRuntimeException(String desc, String ip, int port, String serviceKind) {
+ super(ERROR_CODE, desc, ip, port, serviceKind);
+ }
+
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/AbstractMetaService.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/AbstractMetaService.java
new file mode 100644
index 0000000..744440f
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/AbstractMetaService.java
@@ -0,0 +1,213 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.service;
+
+import com.google.common.cache.Cache;
+import com.webank.wedatasphere.linkis.common.exception.WarnException;
+import com.webank.wedatasphere.linkis.metadatamanager.common.Json;
+import com.webank.wedatasphere.linkis.metadatamanager.common.cache.CacheManager;
+import com.webank.wedatasphere.linkis.metadatamanager.common.cache.ConnCacheManager;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaPartitionInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.exception.MetaRuntimeException;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.PostConstruct;
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Created by jackyxxie on 2020/2/10.
+ */
+public abstract class AbstractMetaService<C extends Closeable> implements MetadataService {
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractMetaService.class);
+ private static final String CONN_CACHE_REQ = "_STORED";
+
+
+ private CacheManager connCacheManager;
+
+ /**
+ * Caching connections which built by connect parameters requested with
+ */
+ protected Cache<String, MetadataConnection<C>> reqCache;
+
+ @PostConstruct
+ public void init(){
+ connCacheManager = ConnCacheManager.custom();
+ initCache(connCacheManager);
+ }
+ /**
+ * If want to use cache component, you should invoke this in constructor method
+ * @param cacheManager
+ */
+ protected void initCache(CacheManager cacheManager){
+ String prefix = this.getClass().getSimpleName();
+ reqCache = cacheManager.buildCache(prefix + CONN_CACHE_REQ, notification ->{
+ assert notification.getValue() != null;
+ close(notification.getValue().getConnection());
+ });
+ }
+
+ @Override
+ public abstract MetadataConnection<C> getConnection(String operator, Map<String, Object> params) throws Exception;
+
+ @Override
+ public List<String> getDatabases(String operator, Map<String, Object> params) {
+ return this.getConnAndRun(operator, params, this::queryDatabases);
+ }
+
+ @Override
+ public List<String> getTables(String operator, Map<String, Object> params, String database) {
+ return this.getConnAndRun(operator, params, conn -> this.queryTables(conn, database));
+ }
+
+ @Override
+ public Map<String, String> getTableProps(String operator, Map<String, Object> params, String database, String table) {
+ return this.getConnAndRun(operator, params, conn -> this.queryTableProps(conn, database, table));
+ }
+
+ @Override
+ public MetaPartitionInfo getPartitions(String operator, Map<String, Object> params, String database, String table) {
+ return this.getConnAndRun(operator, params, conn -> this.queryPartitions(conn, database, table));
+ }
+
+ @Override
+ public List<MetaColumnInfo> getColumns(String operator, Map<String, Object> params, String database, String table) {
+ return this.getConnAndRun(operator, params, conn -> this.queryColumns(conn, database, table));
+ }
+
+ /**
+ * Get database list by connection
+ * @param connection metadata connection
+ * @return
+ */
+ public List<String> queryDatabases(C connection){
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get table list by connection and database
+ * @param connection metadata connection
+ * @param database database
+ * @return
+ */
+ public List<String> queryTables(C connection, String database){
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get partitions by connection, database and table
+ * @param connection metadata connection
+ * @param database database
+ * @param table table
+ * @return
+ */
+ public MetaPartitionInfo queryPartitions(C connection, String database, String table){
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get columns by connection, database and table
+ * @param connection metadata connection
+ * @param database database
+ * @param table table
+ * @return
+ */
+ public List<MetaColumnInfo> queryColumns(C connection, String database, String table){
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ /**
+ * Get table properties
+ * @param connection metadata connection
+ * @param database database
+ * @param table table
+ * @return
+ */
+ public Map<String, String> queryTableProps(C connection, String database, String table){
+ throw new WarnException(-1, "This method is no supported");
+ }
+
+ protected void close(C connection){
+ try {
+ connection.close();
+ } catch (IOException e) {
+ throw new MetaRuntimeException("Fail to close connection[关闭连接失败], [" + e.getMessage() + "]");
+ }
+ }
+
+ protected <R>R getConnAndRun(String operator, Map<String, Object> params, Function<C, R> action) {
+ String cacheKey = "";
+ try{
+ cacheKey = md5String(Json.toJson(params, null), "", 2);
+ MetadataConnection<C> connection;
+ if(null != reqCache) {
+ connection = reqCache.get(cacheKey, () -> getConnection(operator, params));
+ }else{
+ connection = getConnection(operator, params);
+ }
+ return run(connection, action);
+ }catch(Exception e){
+ LOG.error("Error to invoke meta service", e);
+ if(StringUtils.isNotBlank(cacheKey)){
+ reqCache.invalidate(cacheKey);
+ }
+ throw new MetaRuntimeException(e.getMessage());
+ }
+ }
+ private <R>R run(MetadataConnection<C> connection, Function<C, R> action){
+ if(connection.isLock()){
+ connection.getLock().lock();
+ try{
+ return action.apply(connection.getConnection());
+ }finally{
+ connection.getLock().unlock();
+ }
+ }else{
+ return action.apply(connection.getConnection());
+ }
+ }
+
+ private String md5String(String source, String salt, int iterator){
+ StringBuilder token = new StringBuilder();
+ try{
+ MessageDigest digest = MessageDigest.getInstance("md5");
+ if(StringUtils.isNotEmpty(salt)){
+ digest.update(salt.getBytes(StandardCharsets.UTF_8));
+ }
+ byte[] result = digest.digest(source.getBytes());
+ for(int i = 0; i < iterator - 1; i++){
+ digest.reset();
+ result = digest.digest(result);
+ }
+ for (byte aResult : result) {
+ int temp = aResult & 0xFF;
+ if (temp <= 0xF) {
+ token.append("0");
+ }
+ token.append(Integer.toHexString(temp));
+ }
+ }catch(Exception e){
+ throw new RuntimeException(e.getMessage());
+ }
+ return token.toString();
+ }
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/BaseMetadataService.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/BaseMetadataService.java
new file mode 100644
index 0000000..e9146d4
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/BaseMetadataService.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.service;
+
+
+import java.io.Closeable;
+import java.util.Map;
+
+/**
+ * Created by jackyxxie on 2020/02/10.
+ */
+public interface BaseMetadataService {
+
+ /**
+ * Get connection
+ *
+ * @param params connect params
+ * @return
+ */
+ MetadataConnection<? extends Closeable> getConnection(String operator, Map<String, Object> params) throws Exception;
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataConnection.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataConnection.java
new file mode 100644
index 0000000..80a2d9d
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataConnection.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.service;
+
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Connection for metadata
+ * Created by jackyxxie on 2020/2/10.
+ */
+public class MetadataConnection<C> {
+ /***
+ * Avoid complication when different threads calling connection API
+ */
+ private ReentrantLock lock = new ReentrantLock();
+
+ private boolean isLock = true;
+
+ private C connection;
+
+ public MetadataConnection(C connection){
+ this.connection = connection;
+ }
+
+ public MetadataConnection(C connection, boolean isLock){
+ this.connection = connection;
+ this.isLock = isLock;
+ }
+ public ReentrantLock getLock() {
+ return lock;
+ }
+
+ public C getConnection() {
+ return connection;
+ }
+
+ public boolean isLock() {
+ return isLock;
+ }
+
+ public void setLock(boolean lock) {
+ isLock = lock;
+ }
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataDbService.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataDbService.java
new file mode 100644
index 0000000..b4de097
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataDbService.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.service;
+
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaPartitionInfo;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by jackyxxie on 2020/2/10.
+ */
+public interface MetadataDbService extends BaseMetadataService {
+
+ /**
+ * Get all databases
+ * @param params connect params
+ * @return
+ */
+ List<String> getDatabases(String operator, Map<String, Object> params);
+
+ /**
+ * Get all tables from database specified
+ * @param params params
+ * @param database database name
+ * @return
+ */
+ List<String> getTables(String operator, Map<String, Object> params, String database);
+
+ /**
+ * Get table properties from database specified
+ * @param params params
+ * @param database database name
+ * @return
+ */
+ Map<String, String> getTableProps(String operator, Map<String, Object> params, String database, String table);
+ /**
+ * Get all partitions from table specified
+ * @param params params
+ * @param database
+ * @param table
+ * @return
+ */
+ MetaPartitionInfo getPartitions(String operator, Map<String, Object> params, String database, String table);
+
+ /**
+ * Get all field information from table specified
+ * @param params
+ * @param database
+ * @param table
+ * @return
+ */
+ List<MetaColumnInfo> getColumns(String operator, Map<String, Object> params, String database, String table);
+
+}
diff --git a/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataService.java b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataService.java
new file mode 100644
index 0000000..3152182
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/common/service/MetadataService.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.service;
+
+/**
+ * Created by jackyxxie on 2020/2/10.
+ */
+public interface MetadataService extends MetadataDbService {
+}
diff --git a/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataOperateProtocol.scala b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataOperateProtocol.scala
new file mode 100644
index 0000000..834dd26
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataOperateProtocol.scala
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.protocol
+
+import java.util
+
+trait MetadataOperateProtocol {
+
+}
+
+/**
+ * Request to do connect
+ * @param version
+ * @param params
+ */
+case class MetadataConnect(operator: String, params: util.Map[String, Object], version: String) extends MetadataOperateProtocol
diff --git a/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataProtocol.scala b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataProtocol.scala
new file mode 100644
index 0000000..38940e2
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataProtocol.scala
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.protocol
+
+trait MetadataProtocol {
+
+}
+
+case class MetadataResponse(status: Boolean, data: String) extends MetadataProtocol
diff --git a/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataQueryProtocol.scala b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataQueryProtocol.scala
new file mode 100644
index 0000000..2668989
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/protocol/MetadataQueryProtocol.scala
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.protocol
+
+import java.util
+
+trait MetadataQueryProtocol {
+
+}
+
+/**
+ * Request to get database list
+ * @param params
+ */
+case class MetaGetDatabases(params: util.Map[String, Object], operator: String) extends MetadataQueryProtocol
+
+/**
+ * Request to get table list
+ * @param params
+ */
+case class MetaGetTables(params: util.Map[String, Object], database: String, operator: String) extends MetadataQueryProtocol
+
+/**
+ * Request to get table properties
+ * @param params
+ * @param database
+ * @param table
+ */
+case class MetaGetTableProps(params: util.Map[String, Object], database: String, table: String, operator: String) extends MetadataQueryProtocol
+
+/**
+ * Request to get partition list
+ * @param params
+ * @param database
+ * @param table
+ */
+case class MetaGetPartitions(params: util.Map[String, Object], database: String, table: String, operator: String) extends MetadataQueryProtocol
+
+/**
+ * Request to get column list
+ * @param params
+ * @param database
+ * @param table
+ */
+case class MetaGetColumns(params: util.Map[String, Object], database: String, table: String, operator: String) extends MetadataQueryProtocol
+
diff --git a/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/receiver/BaseMetaReceiver.scala b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/receiver/BaseMetaReceiver.scala
new file mode 100644
index 0000000..d4ecf7d
--- /dev/null
+++ b/datasource/metadatamanager/common/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/common/receiver/BaseMetaReceiver.scala
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.common.receiver
+
+import com.webank.wedatasphere.linkis.common.exception.WarnException
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.metadatamanager.common.protocol._
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService
+import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender}
+import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
+
+import scala.concurrent.duration.Duration
+
+/**
+ * Created by jackyxxie on 2020/2/10.
+ */
+class BaseMetaReceiver extends Receiver with Logging{
+ protected var metadataService: MetadataService = _
+
+ override def receive(message: Any, sender: Sender): Unit = ???
+
+ override def receiveAndReply(message: Any, sender: Sender): Any = invoke(metadataService, message)
+
+ override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = invoke(metadataService, message)
+
+
+ def invoke(service: MetadataService, message: Any): Any = Utils.tryCatch{
+ val data = message match {
+ case MetaGetDatabases(params, operator) => service.getDatabases(operator, params)
+ case MetaGetTableProps(params, database, table, operator) => service.getTableProps(operator, params, database, table)
+ case MetaGetTables(params, database, operator) => service.getTables(operator, params, database)
+ case MetaGetPartitions(params, database, table, operator) => service.getPartitions(operator, params, database, table)
+ case MetaGetColumns(params, database, table, operator) => service.getColumns(operator, params, database, table)
+ case MetadataConnect(operator, params, version) =>
+ service.getConnection(operator, params)
+ //MetadataConnection is not scala class
+ null
+ case _ => new Object()
+ }
+ MetadataResponse(status = true, BDPJettyServerHelper.gson.toJson(data))
+ }{
+ case e:WarnException => val errorMsg = e.getMessage
+ info(s"Fail to invoke meta service: [$message],[$errorMsg]")
+ MetadataResponse(status = false, errorMsg)
+ case t:Throwable =>
+ val errorMsg = t.getMessage
+ if (message.isInstanceOf[MetadataConnect])
+ info(s"Fail to invoke meta service: [$message], [$errorMsg]")
+ else error(s"Fail to invoke meta service", t)
+ MetadataResponse(status = false, t.getMessage)
+ }
+}
diff --git a/datasource/metadatamanager/server/bin/start-mdm-server.sh b/datasource/metadatamanager/server/bin/start-mdm-server.sh
new file mode 100644
index 0000000..80cc775
--- /dev/null
+++ b/datasource/metadatamanager/server/bin/start-mdm-server.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+export SERVER_LOG_PATH=$HOME/logs
+export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication
+
+if test -z "$SERVER_HEAP_SIZE"
+then
+ export SERVER_HEAP_SIZE="512M"
+fi
+
+if test -z "$SERVER_JAVA_OPTS"
+then
+ export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log"
+fi
+
+if [[ -f "${SERVER_PID}" ]]; then
+ pid=$(cat ${SERVER_PID})
+ if kill -0 ${pid} >/dev/null 2>&1; then
+ echo "Server is already running."
+ exit 1
+ fi
+fi
+
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+pid=$!
+if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME start failed!"
+ exit 1
+else
+ echo "server $SERVER_NAME start succeeded!"
+ echo $pid > $SERVER_PID
+ sleep 1
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/server/bin/stop-mdm-server.sh b/datasource/metadatamanager/server/bin/stop-mdm-server.sh
new file mode 100644
index 0000000..f032887
--- /dev/null
+++ b/datasource/metadatamanager/server/bin/stop-mdm-server.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+
+function wait_for_server_to_die() {
+ local pid
+ local count
+ pid=$1
+ timeout=$2
+ count=0
+ timeoutTime=$(date "+%s")
+ let "timeoutTime+=$timeout"
+ currentTime=$(date "+%s")
+ forceKill=1
+
+ while [[ $currentTime -lt $timeoutTime ]]; do
+ $(kill ${pid} > /dev/null 2> /dev/null)
+ if kill -0 ${pid} > /dev/null 2>&1; then
+ sleep 3
+ else
+ forceKill=0
+ break
+ fi
+ currentTime=$(date "+%s")
+ done
+
+ if [[ forceKill -ne 0 ]]; then
+ $(kill -9 ${pid} > /dev/null 2> /dev/null)
+ fi
+}
+
+if [[ ! -f "${SERVER_PID}" ]]; then
+ echo "server $SERVER_NAME is not running"
+else
+ pid=$(cat ${SERVER_PID})
+ if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME is not running"
+ else
+ wait_for_server_to_die $pid 40
+ $(rm -f ${SERVER_PID})
+ echo "server $SERVER_NAME is stopped."
+ fi
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/server/conf/application.yml b/datasource/metadatamanager/server/conf/application.yml
new file mode 100644
index 0000000..ad57812
--- /dev/null
+++ b/datasource/metadatamanager/server/conf/application.yml
@@ -0,0 +1,29 @@
+server:
+ port: 8296
+spring:
+ application:
+ name: mdm-server
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://${ip}:${port}/eureka/
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
+
+
+pagehelper:
+ helper-dialect: mysql
+ reasonable: true
+ support-methods-arguments: true
+ params: countSql
+
diff --git a/datasource/metadatamanager/server/conf/linkis.properties b/datasource/metadatamanager/server/conf/linkis.properties
new file mode 100644
index 0000000..4cb512f
--- /dev/null
+++ b/datasource/metadatamanager/server/conf/linkis.properties
@@ -0,0 +1,25 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.server.mybatis.mapperLocations=
+wds.linkis.server.mybatis.typeAliasesPackage=
+wds.linkis.server.mybatis.BasePackage=
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.metadatamanager.server.restful
+
+#sit
+wds.linkis.server.version=v1
+
+#test
+wds.linkis.test.mode=true
+wds.linkis.test.user=davidhua
+
diff --git a/datasource/metadatamanager/server/conf/log4j.properties b/datasource/metadatamanager/server/conf/log4j.properties
new file mode 100644
index 0000000..d5ee44b
--- /dev/null
+++ b/datasource/metadatamanager/server/conf/log4j.properties
@@ -0,0 +1,33 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
\ No newline at end of file
diff --git a/datasource/metadatamanager/server/conf/log4j2.xml b/datasource/metadatamanager/server/conf/log4j2.xml
new file mode 100644
index 0000000..1c68190
--- /dev/null
+++ b/datasource/metadatamanager/server/conf/log4j2.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/datasource/metadatamanager/server/pom.xml b/datasource/metadatamanager/server/pom.xml
new file mode 100644
index 0000000..51a2dc2
--- /dev/null
+++ b/datasource/metadatamanager/server/pom.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>linkis-metadatamanager-server</artifactId>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--Metadata common-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-metadatamanager-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <!-- data source manager common dependency-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-datasourcemanager-common</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--rpc-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cloudRPC</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>linkis-mdm-server</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/java</directory>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>**/*.properties</exclude>
+ <exclude>**/application.yml</exclude>
+ <exclude>**/bootstrap.yml</exclude>
+ <exclude>**/log4j2.xml</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
diff --git a/datasource/metadatamanager/server/src/main/assembly/distribution.xml b/datasource/metadatamanager/server/src/main/assembly/distribution.xml
new file mode 100644
index 0000000..56a7f09
--- /dev/null
+++ b/datasource/metadatamanager/server/src/main/assembly/distribution.xml
@@ -0,0 +1,206 @@
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-mdm-server</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>linkis-mdm-server</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>true</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+
+ <!-- <excludes>
+ <exclude>antlr:antlr:jar</exclude>
+ <exclude>aopalliance:aopalliance:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude>
+ <exclude>com.google.code.gson:gson:jar</exclude>
+ <exclude>com.google.guava:guava:jar</exclude>
+ <exclude>com.google.protobuf:protobuf-java:jar</exclude>
+ <exclude>com.netflix.archaius:archaius-core:jar</exclude>
+ <exclude>com.netflix.hystrix:hystrix-core:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-core:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude>
+ <exclude>com.netflix.servo:servo-core:jar</exclude>
+ <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude>
+ <exclude>com.sun.jersey:jersey-client:jar</exclude>
+ <exclude>com.sun.jersey:jersey-core:jar</exclude>
+ <exclude>com.sun.jersey:jersey-server:jar</exclude>
+ <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude>
+ <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils-core:jar</exclude>
+ <exclude>commons-cli:commons-cli:jar</exclude>
+ <exclude>commons-collections:commons-collections:jar</exclude>
+ <exclude>commons-configuration:commons-configuration:jar</exclude>
+ <exclude>commons-daemon:commons-daemon:jar</exclude>
+ <exclude>commons-dbcp:commons-dbcp:jar</exclude>
+ <exclude>commons-digester:commons-digester:jar</exclude>
+ <exclude>commons-io:commons-io:jar</exclude>
+ <exclude>commons-lang:commons-lang:jar</exclude>
+ <exclude>commons-net:commons-net:jar</exclude>
+ <exclude>commons-pool:commons-pool:jar</exclude>
+ <exclude>io.netty:netty:jar</exclude>
+ <exclude>io.netty:netty-all:jar</exclude>
+ <exclude>io.netty:netty-buffer:jar</exclude>
+ <exclude>io.netty:netty-codec:jar</exclude>
+ <exclude>io.netty:netty-codec-http:jar</exclude>
+ <exclude>io.netty:netty-common:jar</exclude>
+ <exclude>io.netty:netty-handler:jar</exclude>
+ <exclude>io.netty:netty-transport:jar</exclude>
+ <exclude>io.netty:netty-transport-native-epoll:jar</exclude>
+ <exclude>io.reactivex:rxjava:jar</exclude>
+ <exclude>io.reactivex:rxnetty:jar</exclude>
+ <exclude>io.reactivex:rxnetty-contexts:jar</exclude>
+ <exclude>io.reactivex:rxnetty-servo:jar</exclude>
+ <exclude>javax.activation:activation:jar</exclude>
+ <exclude>javax.annotation:javax.annotation-api:jar</exclude>
+ <exclude>javax.inject:javax.inject:jar</exclude>
+ <exclude>javax.servlet:javax.servlet-api:jar</exclude>
+ <exclude>javax.servlet.jsp:jsp-api:jar</exclude>
+ <exclude>javax.xml.bind:jaxb-api:jar</exclude>
+ <exclude>javax.xml.stream:stax-api:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude>
+ <exclude>org.antlr:antlr-runtime:jar</exclude>
+ <exclude>org.antlr:stringtemplate:jar</exclude>
+ <exclude>org.apache.commons:commons-compress:jar</exclude>
+ <exclude>org.apache.commons:commons-math3:jar</exclude>
+ <exclude>org.apache.curator:curator-framework:jar</exclude>
+ <exclude>org.apache.curator:curator-recipes:jar</exclude>
+ <exclude>org.apache.directory.api:api-asn1-api:jar</exclude>
+ <exclude>org.apache.directory.api:api-util:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-auth:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-common:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude>
+ <exclude>org.apache.htrace:htrace-core:jar</exclude>
+ <exclude>org.apache.logging.log4j:log4j-api:jar</exclude>
+ <exclude>org.apache.zookeeper:zookeeper:jar</exclude>
+ <exclude>org.aspectj:aspectjweaver:jar</exclude>
+ <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude>
+ <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude>
+ <exclude>org.codehaus.jettison:jettison:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-http:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-io:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-plus:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-security:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-server:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-util:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-xml:jar</exclude>
+ <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude>
+ <exclude>org.hdrhistogram:HdrHistogram:jar</exclude>
+ <exclude>org.json4s:json4s-ast_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-core_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-jackson_2.11:jar</exclude>
+ <exclude>org.jsoup:jsoup:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty-util:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude>
+ <exclude>org.scala-lang:scala-compiler:jar</exclude>
+ <exclude>org.scala-lang:scala-library:jar</exclude>
+ <exclude>org.scala-lang:scala-reflect:jar</exclude>
+ <exclude>org.scala-lang:scalap:jar</exclude>
+ <exclude>org.slf4j:jul-to-slf4j:jar</exclude>
+ <exclude>org.slf4j:slf4j-api:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.security:spring-security-crypto:jar</exclude>
+ <exclude>org.springframework.security:spring-security-rsa:jar</exclude>
+ <exclude>org.springframework:spring-aop:jar</exclude>
+ <exclude>org.springframework:spring-beans:jar</exclude>
+ <exclude>org.springframework:spring-context:jar</exclude>
+ <exclude>org.springframework:spring-core:jar</exclude>
+ <exclude>org.springframework:spring-expression:jar</exclude>
+ <!–<exclude>org.springframework:spring-jcl:jar</exclude>–>
+ <exclude>org.springframework:spring-web:jar</exclude>
+ <exclude>org.tukaani:xz:jar</exclude>
+ <exclude>org.yaml:snakeyaml:jar</exclude>
+ <exclude>xerces:xercesImpl:jar</exclude>
+ <exclude>xml-apis:xml-apis:jar</exclude>
+ <exclude>xmlenc:xmlenc:jar</exclude>
+ </excludes>-->
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/conf</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/bin</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>.</directory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <outputDirectory>logs</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
diff --git a/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/restful/MetadataCoreRestful.java b/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/restful/MetadataCoreRestful.java
new file mode 100644
index 0000000..63791c5
--- /dev/null
+++ b/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/restful/MetadataCoreRestful.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.server.restful;
+
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaPartitionInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.server.service.MetadataAppService;
+import com.webank.wedatasphere.linkis.server.Message;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author alexyang
+ * 2020/02/10
+ */
+@Path("/metadata")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+@Component
+public class MetadataCoreRestful {
+
+ @Autowired
+ private MetadataAppService metadataAppService;
+
+ @GET
+ @Path("/dbs/{data_source_id}")
+ public Response getDatabases(@PathParam("data_source_id")String dataSourceId,
+ @QueryParam("system")String system){
+ try{
+ if(StringUtils.isBlank(system)){
+ return Message.messageToResponse(Message.error("'system' is missing[缺少系统名]"));
+ }
+ List<String> databases = metadataAppService.getDatabasesByDsId(dataSourceId, system);
+ return Message.messageToResponse(Message.ok().data("dbs", databases));
+ }catch(Exception e){
+ return Message.messageToResponse(
+ Message.error("Fail to get database list[获取库信息失败], id:[" + dataSourceId +"], system:[" + system + "]", e));
+ }
+ }
+
+ @GET
+ @Path("/tables/{data_source_id}/db/{database}")
+ public Response getTables(@PathParam("data_source_id")String dataSourceId,
+ @PathParam("database")String database,
+ @QueryParam("system")String system){
+ try{
+ if(StringUtils.isBlank(system)){
+ return Message.messageToResponse(Message.error("'system' is missing[缺少系统名]"));
+ }
+ List<String> tables = metadataAppService.getTablesByDsId(dataSourceId, database, system);
+ return Message.messageToResponse(Message.ok().data("tables", tables));
+ }catch(Exception e){
+ return Message.messageToResponse(
+ Message.error("Fail to get table list[获取表信息失败], id:[" + dataSourceId +"]" +
+ ", system:[" + system + "], database:[" +database +"]", e));
+ }
+ }
+
+ @GET
+ @Path("/props/{data_source_id}/db/{database}/table/{table}")
+ public Response getTableProps(@PathParam("data_source_id")String dataSourceId,
+ @PathParam("database")String database,
+ @PathParam("table") String table,
+ @QueryParam("system")String system){
+ try{
+ if(StringUtils.isBlank(system)){
+ return Message.messageToResponse(Message.error("'system' is missing[缺少系统名]"));
+ }
+ Map<String, String> tableProps = metadataAppService.getTablePropsByDsId(dataSourceId, database, table, system);
+ return Message.messageToResponse(Message.ok().data("props", tableProps));
+ }catch(Exception e){
+ return Message.messageToResponse(
+ Message.error("Fail to get table properties[获取表参数信息失败], id:[" + dataSourceId +"]" +
+ ", system:[" + system + "], database:[" +database +"], table:[" + table +"]", e));
+ }
+ }
+
+ @GET
+ @Path("/partitions/{data_source_id}/db/{database}/table/{table}")
+ public Response getPartitions(@PathParam("data_source_id")String dataSourceId,
+ @PathParam("database")String database,
+ @PathParam("table") String table,
+ @QueryParam("system")String system){
+ try{
+ if(StringUtils.isBlank(system)){
+ return Message.messageToResponse(Message.error("'system' is missing[缺少系统名]"));
+ }
+ MetaPartitionInfo partitionInfo = metadataAppService.getPartitionsByDsId(dataSourceId, database, table, system);
+ return Message.messageToResponse(Message.ok().data("props", partitionInfo));
+ }catch(Exception e){
+ return Message.messageToResponse(
+ Message.error("Fail to get partitions[获取表分区信息失败], id:[" + dataSourceId +"]" +
+ ", system:[" + system + "], database:[" +database +"], table:[" + table +"]"));
+ }
+ }
+
+ @GET
+ @Path("/columns/{data_source_id}/db/{database}/table/{table}")
+ public Response getColumns(@PathParam("data_source_id")String dataSourceId,
+ @PathParam("database")String database,
+ @PathParam("table") String table,
+ @QueryParam("system")String system){
+ try{
+ if(StringUtils.isBlank(system)){
+ return Message.messageToResponse(Message.error("'system' is missing[缺少系统名]"));
+ }
+ List<MetaColumnInfo> columns = metadataAppService.getColumns(dataSourceId, database, table, system);
+ return Message.messageToResponse(Message.ok().data("columns", columns));
+ }catch(Exception e){
+ return Message.messageToResponse(
+ Message.error("Fail to get column list[获取表字段信息失败], id:[" + dataSourceId +"]" +
+ ", system:[" + system + "], database:[" +database +"], table:[" + table +"]", e));
+ }
+ }
+
+}
diff --git a/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/service/MetadataAppService.java b/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/service/MetadataAppService.java
new file mode 100644
index 0000000..1656b94
--- /dev/null
+++ b/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/service/MetadataAppService.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.server.service;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaPartitionInfo;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author alexyang
+ * 2020/02/10
+ */
+public interface MetadataAppService {
+
+ /**
+ * @param dataSourceId data source id
+ * @param system system
+ * @return
+ */
+ List<String> getDatabasesByDsId(String dataSourceId, String system) throws ErrorException;
+
+ /**
+ * @param dataSourceId data source id
+ * @param system system
+ * @param database database
+ * @return
+ */
+ List<String> getTablesByDsId(String dataSourceId, String database, String system) throws ErrorException;
+
+ /**
+ * @param dataSourceId data source id
+ * @param database database
+ * @param table table
+ * @param system system
+ * @return
+ */
+ Map<String, String> getTablePropsByDsId(String dataSourceId, String database, String table, String system) throws ErrorException;
+ /**
+ * @param dataSourceId data source i
+ * @param database database
+ * @param table table
+ * @param system system
+ * @return
+ */
+ MetaPartitionInfo getPartitionsByDsId(String dataSourceId, String database, String table, String system) throws ErrorException;
+
+ /**
+ * @param dataSourceId data source id
+ * @param database database
+ * @param table table
+ * @param system system
+ * @return
+ */
+ List<MetaColumnInfo> getColumns(String dataSourceId, String database, String table, String system) throws ErrorException;
+}
diff --git a/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/service/impl/MetadataAppServiceImpl.java b/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/service/impl/MetadataAppServiceImpl.java
new file mode 100644
index 0000000..371e726
--- /dev/null
+++ b/datasource/metadatamanager/server/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/server/service/impl/MetadataAppServiceImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.server.service.impl;
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.util.json.Json;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.protocol.DsInfoQueryRequest;
+import com.webank.wedatasphere.linkis.datasourcemanager.common.protocol.DsInfoResponse;
+import com.webank.wedatasphere.linkis.metadatamanager.common.MdmConfiguration;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaPartitionInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.server.service.MetadataAppService;
+import com.webank.wedatasphere.linkis.metadatamanager.common.protocol.*;
+import com.webank.wedatasphere.linkis.rpc.Sender;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author alexyang
+ * 2020/02/10
+ */
+@Service
+public class MetadataAppServiceImpl implements MetadataAppService {
+ private Sender dataSourceRpcSender;
+
+ @PostConstruct
+ public void init(){
+ dataSourceRpcSender = Sender.getSender(MdmConfiguration.DATA_SOURCE_SERVICE_APPLICATION.getValue());
+ }
+ @Override
+ public List<String> getDatabasesByDsId(String dataSourceId, String system) throws ErrorException {
+ DsInfoResponse dsInfoResponse = reqToGetDataSourceInfo(dataSourceId, system);
+ if(StringUtils.isNotBlank(dsInfoResponse.dsType())){
+ MetadataResponse metaQueryResponse = doAndGetMetaInfo(dsInfoResponse.dsType(),
+ new MetaGetDatabases(dsInfoResponse.params(), dsInfoResponse.creator()));
+ return Json.fromJson(metaQueryResponse.data(), List.class, String.class);
+ }
+ return new ArrayList<>();
+ }
+
+ @Override
+ public List<String> getTablesByDsId(String dataSourceId, String database, String system) throws ErrorException {
+ DsInfoResponse dsInfoResponse = reqToGetDataSourceInfo(dataSourceId, system);
+ if(StringUtils.isNotBlank(dsInfoResponse.dsType())){
+ MetadataResponse metaQueryResponse = doAndGetMetaInfo(dsInfoResponse.dsType(),
+ new MetaGetTables(dsInfoResponse.params(), database, dsInfoResponse.creator()));
+ return Json.fromJson(metaQueryResponse.data(), List.class, String.class);
+ }
+ return new ArrayList<>();
+ }
+
+ @Override
+ public Map<String, String> getTablePropsByDsId(String dataSourceId, String database, String table, String system) throws ErrorException {
+ DsInfoResponse dsInfoResponse = reqToGetDataSourceInfo(dataSourceId, system);
+ if(StringUtils.isNotBlank(dsInfoResponse.dsType())){
+ MetadataResponse metaQueryResponse = doAndGetMetaInfo(dsInfoResponse.dsType(),
+ new MetaGetTableProps(dsInfoResponse.params(), database, table, dsInfoResponse.creator()));
+ return Json.fromJson(metaQueryResponse.data(), Map.class, String.class, String.class);
+ }
+ return new HashMap<>();
+ }
+
+ @Override
+ public MetaPartitionInfo getPartitionsByDsId(String dataSourceId, String database, String table, String system) throws ErrorException {
+ DsInfoResponse dsInfoResponse = reqToGetDataSourceInfo(dataSourceId, system);
+ if(StringUtils.isNotBlank(dsInfoResponse.dsType())){
+ MetadataResponse metaQueryResponse = doAndGetMetaInfo(dsInfoResponse.dsType(),
+ new MetaGetPartitions(dsInfoResponse.params(), database, table, dsInfoResponse.creator()));
+ return Json.fromJson(metaQueryResponse.data(), MetaPartitionInfo.class);
+ }
+ return new MetaPartitionInfo();
+ }
+
+ @Override
+ public List<MetaColumnInfo> getColumns(String dataSourceId, String database, String table, String system) throws ErrorException {
+ DsInfoResponse dsInfoResponse = reqToGetDataSourceInfo(dataSourceId, system);
+ if(StringUtils.isNotBlank(dsInfoResponse.dsType())){
+ MetadataResponse metaQueryResponse = doAndGetMetaInfo(dsInfoResponse.dsType(),
+ new MetaGetColumns(dsInfoResponse.params(), database, table, dsInfoResponse.creator()));
+ return Json.fromJson(metaQueryResponse.data(), List.class, MetaColumnInfo.class);
+ }
+ return new ArrayList<>();
+ }
+
+ /**
+ * Request to get data source information
+ * (type and connection parameters)
+ * @param dataSourceId data source id
+ * @param system system
+ * @return
+ * @throws ErrorException
+ */
+ public DsInfoResponse reqToGetDataSourceInfo(String dataSourceId, String system) throws ErrorException{
+ Object rpcResult = null;
+ try {
+ rpcResult = dataSourceRpcSender.ask(new DsInfoQueryRequest(dataSourceId, system));
+ }catch(Exception e){
+ throw new ErrorException(-1, "Remote Service Error[远端服务出错, 联系运维处理]");
+ }
+ if(rpcResult instanceof DsInfoResponse){
+ DsInfoResponse response = (DsInfoResponse)rpcResult;
+ if(!response.status()){
+ throw new ErrorException(-1, "Error in Data Source Manager Server[数据源服务出错]");
+ }
+ return response;
+ }else{
+ throw new ErrorException(-1, "Remote Service Error[远端服务出错, 联系运维处理]");
+ }
+ }
+
+ /**
+ * Request to get meta information
+ * @param dataSourceType
+ * @param request
+ * @return
+ */
+ public MetadataResponse doAndGetMetaInfo(String dataSourceType, MetadataQueryProtocol request) throws ErrorException {
+ Sender sender = Sender.getSender(MdmConfiguration.METADATA_SERVICE_APPLICATION.getValue() + "-" + dataSourceType.toLowerCase());
+ Object rpcResult = null;
+ try{
+ rpcResult = sender.ask(request);
+ }catch(Exception e){
+ throw new ErrorException(-1, "Remote Service Error[远端服务出错, 联系运维处理]");
+ }
+ if(rpcResult instanceof MetadataResponse){
+ MetadataResponse response = (MetadataResponse)rpcResult;
+ if(!response.status()){
+ throw new ErrorException(-1, "Error in ["+dataSourceType.toUpperCase()+"] Metadata Service Server[元数据服务出错], " +
+ "[" +response.data() + "]");
+ }
+ return response;
+ }else{
+ throw new ErrorException(-1, "Remote Service Error[远端服务出错, 联系运维处理]");
+ }
+ }
+}
diff --git a/datasource/metadatamanager/service/elasticsearch/bin/start-mdm-elastic.sh b/datasource/metadatamanager/service/elasticsearch/bin/start-mdm-elastic.sh
new file mode 100644
index 0000000..80cc775
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/bin/start-mdm-elastic.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+export SERVER_LOG_PATH=$HOME/logs
+export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication
+
+if test -z "$SERVER_HEAP_SIZE"
+then
+ export SERVER_HEAP_SIZE="512M"
+fi
+
+if test -z "$SERVER_JAVA_OPTS"
+then
+ export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log"
+fi
+
+if [[ -f "${SERVER_PID}" ]]; then
+ pid=$(cat ${SERVER_PID})
+ if kill -0 ${pid} >/dev/null 2>&1; then
+ echo "Server is already running."
+ exit 1
+ fi
+fi
+
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+pid=$!
+if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME start failed!"
+ exit 1
+else
+ echo "server $SERVER_NAME start succeeded!"
+ echo $pid > $SERVER_PID
+ sleep 1
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/elasticsearch/bin/stop-mdm-elastic.sh b/datasource/metadatamanager/service/elasticsearch/bin/stop-mdm-elastic.sh
new file mode 100644
index 0000000..f032887
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/bin/stop-mdm-elastic.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+
+function wait_for_server_to_die() {
+ local pid
+ local count
+ pid=$1
+ timeout=$2
+ count=0
+ timeoutTime=$(date "+%s")
+ let "timeoutTime+=$timeout"
+ currentTime=$(date "+%s")
+ forceKill=1
+
+ while [[ $currentTime -lt $timeoutTime ]]; do
+ $(kill ${pid} > /dev/null 2> /dev/null)
+ if kill -0 ${pid} > /dev/null 2>&1; then
+ sleep 3
+ else
+ forceKill=0
+ break
+ fi
+ currentTime=$(date "+%s")
+ done
+
+ if [[ forceKill -ne 0 ]]; then
+ $(kill -9 ${pid} > /dev/null 2> /dev/null)
+ fi
+}
+
+if [[ ! -f "${SERVER_PID}" ]]; then
+ echo "server $SERVER_NAME is not running"
+else
+ pid=$(cat ${SERVER_PID})
+ if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME is not running"
+ else
+ wait_for_server_to_die $pid 40
+ $(rm -f ${SERVER_PID})
+ echo "server $SERVER_NAME is stopped."
+ fi
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/elasticsearch/conf/application.yml b/datasource/metadatamanager/service/elasticsearch/conf/application.yml
new file mode 100644
index 0000000..8b3f9ce
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/conf/application.yml
@@ -0,0 +1,23 @@
+server:
+ port: 8295
+spring:
+ application:
+ name: mdm-service-elasticsearch
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://${ip}:${port}/eureka/
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
+
+
diff --git a/datasource/metadatamanager/service/elasticsearch/conf/linkis.properties b/datasource/metadatamanager/service/elasticsearch/conf/linkis.properties
new file mode 100644
index 0000000..f83c10d
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/conf/linkis.properties
@@ -0,0 +1,21 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.server.mybatis.mapperLocations=
+wds.linkis.server.mybatis.typeAliasesPackage=
+wds.linkis.server.mybatis.BasePackage=
+wds.linkis.server.restful.scan.packages=
+
+#sit
+wds.linkis.server.version=v1
+
diff --git a/datasource/metadatamanager/service/elasticsearch/conf/log4j.properties b/datasource/metadatamanager/service/elasticsearch/conf/log4j.properties
new file mode 100644
index 0000000..d5ee44b
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/conf/log4j.properties
@@ -0,0 +1,33 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/elasticsearch/conf/log4j2.xml b/datasource/metadatamanager/service/elasticsearch/conf/log4j2.xml
new file mode 100644
index 0000000..1c68190
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/conf/log4j2.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/datasource/metadatamanager/service/elasticsearch/pom.xml b/datasource/metadatamanager/service/elasticsearch/pom.xml
new file mode 100644
index 0000000..9ac98d0
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/pom.xml
@@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-metadatamanager-service-es</artifactId>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <es.version>6.7.1</es.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-metadatamanager-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>elasticsearch-rest-client</artifactId>
+ <version>${es.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>linkis-mdm-service-es</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/java</directory>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>**/*.properties</exclude>
+ <exclude>**/application.yml</exclude>
+ <exclude>**/bootstrap.yml</exclude>
+ <exclude>**/log4j2.xml</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/assembly/distribution.xml b/datasource/metadatamanager/service/elasticsearch/src/main/assembly/distribution.xml
new file mode 100644
index 0000000..f95f84e
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/src/main/assembly/distribution.xml
@@ -0,0 +1,206 @@
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-mdm-service-es</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>linkis-mdm-service-es</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>true</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+
+ <!-- <excludes>
+ <exclude>antlr:antlr:jar</exclude>
+ <exclude>aopalliance:aopalliance:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude>
+ <exclude>com.google.code.gson:gson:jar</exclude>
+ <exclude>com.google.guava:guava:jar</exclude>
+ <exclude>com.google.protobuf:protobuf-java:jar</exclude>
+ <exclude>com.netflix.archaius:archaius-core:jar</exclude>
+ <exclude>com.netflix.hystrix:hystrix-core:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-core:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude>
+ <exclude>com.netflix.servo:servo-core:jar</exclude>
+ <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude>
+ <exclude>com.sun.jersey:jersey-client:jar</exclude>
+ <exclude>com.sun.jersey:jersey-core:jar</exclude>
+ <exclude>com.sun.jersey:jersey-server:jar</exclude>
+ <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude>
+ <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils-core:jar</exclude>
+ <exclude>commons-cli:commons-cli:jar</exclude>
+ <exclude>commons-collections:commons-collections:jar</exclude>
+ <exclude>commons-configuration:commons-configuration:jar</exclude>
+ <exclude>commons-daemon:commons-daemon:jar</exclude>
+ <exclude>commons-dbcp:commons-dbcp:jar</exclude>
+ <exclude>commons-digester:commons-digester:jar</exclude>
+ <exclude>commons-io:commons-io:jar</exclude>
+ <exclude>commons-lang:commons-lang:jar</exclude>
+ <exclude>commons-net:commons-net:jar</exclude>
+ <exclude>commons-pool:commons-pool:jar</exclude>
+ <exclude>io.netty:netty:jar</exclude>
+ <exclude>io.netty:netty-all:jar</exclude>
+ <exclude>io.netty:netty-buffer:jar</exclude>
+ <exclude>io.netty:netty-codec:jar</exclude>
+ <exclude>io.netty:netty-codec-http:jar</exclude>
+ <exclude>io.netty:netty-common:jar</exclude>
+ <exclude>io.netty:netty-handler:jar</exclude>
+ <exclude>io.netty:netty-transport:jar</exclude>
+ <exclude>io.netty:netty-transport-native-epoll:jar</exclude>
+ <exclude>io.reactivex:rxjava:jar</exclude>
+ <exclude>io.reactivex:rxnetty:jar</exclude>
+ <exclude>io.reactivex:rxnetty-contexts:jar</exclude>
+ <exclude>io.reactivex:rxnetty-servo:jar</exclude>
+ <exclude>javax.activation:activation:jar</exclude>
+ <exclude>javax.annotation:javax.annotation-api:jar</exclude>
+ <exclude>javax.inject:javax.inject:jar</exclude>
+ <exclude>javax.servlet:javax.servlet-api:jar</exclude>
+ <exclude>javax.servlet.jsp:jsp-api:jar</exclude>
+ <exclude>javax.xml.bind:jaxb-api:jar</exclude>
+ <exclude>javax.xml.stream:stax-api:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude>
+ <exclude>org.antlr:antlr-runtime:jar</exclude>
+ <exclude>org.antlr:stringtemplate:jar</exclude>
+ <exclude>org.apache.commons:commons-compress:jar</exclude>
+ <exclude>org.apache.commons:commons-math3:jar</exclude>
+ <exclude>org.apache.curator:curator-framework:jar</exclude>
+ <exclude>org.apache.curator:curator-recipes:jar</exclude>
+ <exclude>org.apache.directory.api:api-asn1-api:jar</exclude>
+ <exclude>org.apache.directory.api:api-util:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-auth:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-common:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude>
+ <exclude>org.apache.htrace:htrace-core:jar</exclude>
+ <exclude>org.apache.logging.log4j:log4j-api:jar</exclude>
+ <exclude>org.apache.zookeeper:zookeeper:jar</exclude>
+ <exclude>org.aspectj:aspectjweaver:jar</exclude>
+ <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude>
+ <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude>
+ <exclude>org.codehaus.jettison:jettison:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-http:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-io:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-plus:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-security:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-server:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-util:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-xml:jar</exclude>
+ <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude>
+ <exclude>org.hdrhistogram:HdrHistogram:jar</exclude>
+ <exclude>org.json4s:json4s-ast_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-core_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-jackson_2.11:jar</exclude>
+ <exclude>org.jsoup:jsoup:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty-util:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude>
+ <exclude>org.scala-lang:scala-compiler:jar</exclude>
+ <exclude>org.scala-lang:scala-library:jar</exclude>
+ <exclude>org.scala-lang:scala-reflect:jar</exclude>
+ <exclude>org.scala-lang:scalap:jar</exclude>
+ <exclude>org.slf4j:jul-to-slf4j:jar</exclude>
+ <exclude>org.slf4j:slf4j-api:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.security:spring-security-crypto:jar</exclude>
+ <exclude>org.springframework.security:spring-security-rsa:jar</exclude>
+ <exclude>org.springframework:spring-aop:jar</exclude>
+ <exclude>org.springframework:spring-beans:jar</exclude>
+ <exclude>org.springframework:spring-context:jar</exclude>
+ <exclude>org.springframework:spring-core:jar</exclude>
+ <exclude>org.springframework:spring-expression:jar</exclude>
+ <!–<exclude>org.springframework:spring-jcl:jar</exclude>–>
+ <exclude>org.springframework:spring-web:jar</exclude>
+ <exclude>org.tukaani:xz:jar</exclude>
+ <exclude>org.yaml:snakeyaml:jar</exclude>
+ <exclude>xerces:xercesImpl:jar</exclude>
+ <exclude>xml-apis:xml-apis:jar</exclude>
+ <exclude>xmlenc:xmlenc:jar</exclude>
+ </excludes>-->
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/conf</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/bin</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>.</directory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <outputDirectory>logs</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticConnection.java b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticConnection.java
new file mode 100644
index 0000000..b485c69
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticConnection.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.metadatamanager.common.Json;
+import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.nio.reactor.IOReactorConfig;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+public class ElasticConnection implements Closeable {
+
+ public static final String DEFAULT_TYPE_NAME = "type";
+
+ private static final String DEFAULT_MAPPING_NAME = "mappings";
+ private static final String DEFAULT_INDEX_NAME = "index";
+ private static final String FIELD_PROPS = "properties";
+
+ private RestClient restClient;
+
+ public ElasticConnection(String[] endPoints, String username, String password) throws IOException {
+ HttpHost[] httpHosts = new HttpHost[endPoints.length];
+ for(int i = 0; i < endPoints.length; i++){
+ httpHosts[i] = HttpHost.create(endPoints[i]);
+ }
+ RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
+ CredentialsProvider credentialsProvider = null;
+ if(StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)){
+ credentialsProvider = new BasicCredentialsProvider();
+ credentialsProvider.setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(username, password));
+ }
+ //set only one thread
+ CredentialsProvider finalCredentialsProvider = credentialsProvider;
+ restClientBuilder.setHttpClientConfigCallback(httpClientBuilder -> {
+ if(null != finalCredentialsProvider){
+ httpClientBuilder
+ .setDefaultCredentialsProvider(finalCredentialsProvider);
+ }
+ return httpClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build());
+ });
+ this.restClient = restClientBuilder.build();
+ //Try to test connection
+ ping();
+ }
+
+ public List<String> getAllIndices() throws IOException {
+ List<String> indices = new ArrayList<>();
+ Request request = new Request("GET", "_cat/indices");
+ request.addParameter("format", "JSON");
+ Response response = restClient.performRequest(request);
+ List<Map<String, Object>> list = Json.fromJson(response.getEntity().getContent(), Map.class);
+ list.forEach( v ->{
+ String index = String.valueOf(v.getOrDefault(DEFAULT_INDEX_NAME, ""));
+ if(StringUtils.isNotBlank(index) && !index.startsWith(".")){
+ indices.add(index);
+ }
+ });
+ return indices;
+ }
+
+ public List<String> getTypes(String index) throws IOException{
+ List<String> types = new ArrayList<>();
+ Request request = new Request("GET", index +"/_mappings");
+ Response response = restClient.performRequest(request);
+ Map<String, Map<String, Object>> result =
+ Json.fromJson(response.getEntity().getContent(), Map.class);
+ Map<String, Object> indexMap = result.get(index);
+ Object props = indexMap.get(DEFAULT_MAPPING_NAME);
+ if(props instanceof Map){
+ Set keySet = ((Map)props).keySet();
+ for(Object v : keySet){
+ types.add(String.valueOf(v));
+ }
+ }
+ return types;
+ }
+
+ public Map<Object, Object> getProps(String index, String type) throws IOException{
+ Request request = new Request("GET", index + "/_mappings/" + type);
+ Response response = restClient.performRequest(request);
+ Map<String, Map<String, Object>> result =
+ Json.fromJson(response.getEntity().getContent(), Map.class);
+ Map mappings = (Map)result.get(index).get("mappings");
+ Map propsMap = mappings;
+ if(mappings.containsKey(type)){
+ Object typeMap = mappings.get(type);
+ if(typeMap instanceof Map){
+ propsMap = (Map)typeMap;
+ }
+ }
+ Object props = propsMap.get(FIELD_PROPS);
+ if(props instanceof Map){
+ return (Map)props;
+ }
+ return null;
+ }
+ public void ping() throws IOException{
+ Response response = restClient.performRequest(new Request("GET", "/"));
+ int code = response.getStatusLine().getStatusCode();
+ int successCode = 200;
+ if(code != successCode){
+ throw new RuntimeException("Ping to ElasticSearch ERROR, response code: " + code);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.restClient.close();
+ }
+}
diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticMetaService.java b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticMetaService.java
new file mode 100644
index 0000000..8778ba4
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticMetaService.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.metadatamanager.common.Json;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.AbstractMetaService;
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataConnection;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+@Component
+public class ElasticMetaService extends AbstractMetaService<ElasticConnection> {
+ @Override
+ public MetadataConnection<ElasticConnection> getConnection(String operator, Map<String, Object> params)
+ throws Exception{
+ String[] endPoints = new String[]{};
+ Object urls = params.get(ElasticParamsMapper.PARAM_ES_URLS.getValue());
+ if(!(urls instanceof List)){
+ List<String> urlList = Json.fromJson(String.valueOf(urls), List.class, String.class);
+ assert urlList != null;
+ endPoints = urlList.toArray(endPoints);
+ }else{
+ endPoints = ((List<String>)urls).toArray(endPoints);
+ }
+ ElasticConnection conn = new ElasticConnection(endPoints, String.valueOf(params.getOrDefault(ElasticParamsMapper.PARAM_ES_USERNAME.getValue(), "")),
+ String.valueOf(params.getOrDefault(ElasticParamsMapper.PARAM_ES_PASSWORD.getValue(), "")));
+ return new MetadataConnection<>(conn, false);
+ }
+
+ @Override
+ public List<String> queryDatabases(ElasticConnection connection) {
+ //Get indices
+ try{
+ return connection.getAllIndices();
+ }catch (Exception e){
+ throw new RuntimeException("Fail to get ElasticSearch indices(获取索引列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<String> queryTables(ElasticConnection connection, String database) {
+ //Get types
+ try{
+ return connection.getTypes(database);
+ }catch (Exception e){
+ throw new RuntimeException("Fail to get ElasticSearch types(获取索引类型失败)", e);
+ }
+ }
+
+ @Override
+ public List<MetaColumnInfo> queryColumns(ElasticConnection connection, String database, String table) {
+ try {
+ Map<Object, Object> props = connection.getProps(database, table);
+ return props.entrySet().stream().map(entry -> {
+ MetaColumnInfo info = new MetaColumnInfo();
+ info.setName(String.valueOf(entry.getKey()));
+ Object value = entry.getValue();
+ if(value instanceof Map){
+ info.setType(String.valueOf(((Map)value)
+ .getOrDefault(ElasticConnection.DEFAULT_TYPE_NAME, "")));
+ }
+ return info;
+ }).collect(Collectors.toList());
+ } catch (Exception e) {
+ throw new RuntimeException("Fail to get ElasticSearch columns(获取索引字段失败)", e);
+ }
+ }
+}
diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticParamsMapper.java b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticParamsMapper.java
new file mode 100644
index 0000000..3af7e8a
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/ElasticParamsMapper.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * Configuration
+ * @author davidhua
+ * 2020/02/14
+ */
+public class ElasticParamsMapper {
+ public static final CommonVars<String> PARAM_ES_URLS =
+ CommonVars.apply("wds.linkis.server.mdm.service.es.urls", "elasticUrls");
+
+ public static final CommonVars<String> PARAM_ES_USERNAME =
+ CommonVars.apply("wds.linkis.server.mdm.service.es.username", "username");
+
+ public static final CommonVars<String> PARAM_ES_PASSWORD =
+ CommonVars.apply("wds.linkis.server.mdm.service.es.password", "password");
+}
diff --git a/datasource/metadatamanager/service/elasticsearch/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/ElasticReceiver.scala b/datasource/metadatamanager/service/elasticsearch/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/ElasticReceiver.scala
new file mode 100644
index 0000000..dd3049f
--- /dev/null
+++ b/datasource/metadatamanager/service/elasticsearch/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/ElasticReceiver.scala
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service.receiver
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService
+import com.webank.wedatasphere.linkis.metadatamanager.common.receiver.BaseMetaReceiver
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService
+import javax.annotation.PostConstruct
+import org.springframework.stereotype.Component
+
+@Component
+class ElasticReceiver extends BaseMetaReceiver{
+ @PostConstruct
+ def init(): Unit = {
+ metadataService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[MetadataService])
+ }
+}
diff --git a/datasource/metadatamanager/service/hive/bin/start-mdm-hive.sh b/datasource/metadatamanager/service/hive/bin/start-mdm-hive.sh
new file mode 100644
index 0000000..80cc775
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/bin/start-mdm-hive.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+export SERVER_LOG_PATH=$HOME/logs
+export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication
+
+if test -z "$SERVER_HEAP_SIZE"
+then
+ export SERVER_HEAP_SIZE="512M"
+fi
+
+if test -z "$SERVER_JAVA_OPTS"
+then
+ export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log"
+fi
+
+if [[ -f "${SERVER_PID}" ]]; then
+ pid=$(cat ${SERVER_PID})
+ if kill -0 ${pid} >/dev/null 2>&1; then
+ echo "Server is already running."
+ exit 1
+ fi
+fi
+
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+pid=$!
+if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME start failed!"
+ exit 1
+else
+ echo "server $SERVER_NAME start succeeded!"
+ echo $pid > $SERVER_PID
+ sleep 1
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/hive/bin/stop-mdm-hive.sh b/datasource/metadatamanager/service/hive/bin/stop-mdm-hive.sh
new file mode 100644
index 0000000..f032887
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/bin/stop-mdm-hive.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+
+function wait_for_server_to_die() {
+ local pid
+ local count
+ pid=$1
+ timeout=$2
+ count=0
+ timeoutTime=$(date "+%s")
+ let "timeoutTime+=$timeout"
+ currentTime=$(date "+%s")
+ forceKill=1
+
+ while [[ $currentTime -lt $timeoutTime ]]; do
+ $(kill ${pid} > /dev/null 2> /dev/null)
+ if kill -0 ${pid} > /dev/null 2>&1; then
+ sleep 3
+ else
+ forceKill=0
+ break
+ fi
+ currentTime=$(date "+%s")
+ done
+
+ if [[ forceKill -ne 0 ]]; then
+ $(kill -9 ${pid} > /dev/null 2> /dev/null)
+ fi
+}
+
+if [[ ! -f "${SERVER_PID}" ]]; then
+ echo "server $SERVER_NAME is not running"
+else
+ pid=$(cat ${SERVER_PID})
+ if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME is not running"
+ else
+ wait_for_server_to_die $pid 40
+ $(rm -f ${SERVER_PID})
+ echo "server $SERVER_NAME is stopped."
+ fi
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/hive/conf/application.yml b/datasource/metadatamanager/service/hive/conf/application.yml
new file mode 100644
index 0000000..1b52e76
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/conf/application.yml
@@ -0,0 +1,23 @@
+server:
+ port: 8293
+spring:
+ application:
+ name: mdm-service-hive
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://${ip}:${port}/eureka/
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
+
+
diff --git a/datasource/metadatamanager/service/hive/conf/linkis.properties b/datasource/metadatamanager/service/hive/conf/linkis.properties
new file mode 100644
index 0000000..edea49c
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/conf/linkis.properties
@@ -0,0 +1,24 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.server.mybatis.mapperLocations=
+wds.linkis.server.mybatis.typeAliasesPackage=
+wds.linkis.server.mybatis.BasePackage=
+wds.linkis.server.restful.scan.packages=
+
+#sit
+wds.linkis.server.version=v1
+
+#bml
+wds.linkis.gateway.ip=
+wds.linkis.gateway.port=
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/hive/conf/log4j.properties b/datasource/metadatamanager/service/hive/conf/log4j.properties
new file mode 100644
index 0000000..d5ee44b
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/conf/log4j.properties
@@ -0,0 +1,33 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/hive/conf/log4j2.xml b/datasource/metadatamanager/service/hive/conf/log4j2.xml
new file mode 100644
index 0000000..1c68190
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/conf/log4j2.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/datasource/metadatamanager/service/hive/pom.xml b/datasource/metadatamanager/service/hive/pom.xml
new file mode 100644
index 0000000..20a3880
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/pom.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>linkis-metadatamanager-service-hive</artifactId>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <hive.version>1.2.1</hive.version>
+ <hadoop.version>2.7.2</hadoop.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-metadatamanager-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <!--Hive dependencies-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>servlet-api</artifactId>
+ <groupId>javax.servlet</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>guava</artifactId>
+ <groupId>com.google.guava</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ivy</groupId>
+ <artifactId>ivy</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>guava</artifactId>
+ <groupId>com.google.guava</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>commons-lang3</artifactId>
+ <groupId>org.apache.commons</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--bml client-->
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-bmlclient</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>linkis-mdm-service-hive</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/java</directory>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>**/*.properties</exclude>
+ <exclude>**/application.yml</exclude>
+ <exclude>**/bootstrap.yml</exclude>
+ <exclude>**/log4j2.xml</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
diff --git a/datasource/metadatamanager/service/hive/src/main/assembly/distribution.xml b/datasource/metadatamanager/service/hive/src/main/assembly/distribution.xml
new file mode 100644
index 0000000..234308b
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/src/main/assembly/distribution.xml
@@ -0,0 +1,206 @@
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-mdm-service-hive</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>linkis-mdm-service-hive</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>true</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+
+ <!-- <excludes>
+ <exclude>antlr:antlr:jar</exclude>
+ <exclude>aopalliance:aopalliance:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude>
+ <exclude>com.google.code.gson:gson:jar</exclude>
+ <exclude>com.google.guava:guava:jar</exclude>
+ <exclude>com.google.protobuf:protobuf-java:jar</exclude>
+ <exclude>com.netflix.archaius:archaius-core:jar</exclude>
+ <exclude>com.netflix.hystrix:hystrix-core:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-core:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude>
+ <exclude>com.netflix.servo:servo-core:jar</exclude>
+ <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude>
+ <exclude>com.sun.jersey:jersey-client:jar</exclude>
+ <exclude>com.sun.jersey:jersey-core:jar</exclude>
+ <exclude>com.sun.jersey:jersey-server:jar</exclude>
+ <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude>
+ <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils-core:jar</exclude>
+ <exclude>commons-cli:commons-cli:jar</exclude>
+ <exclude>commons-collections:commons-collections:jar</exclude>
+ <exclude>commons-configuration:commons-configuration:jar</exclude>
+ <exclude>commons-daemon:commons-daemon:jar</exclude>
+ <exclude>commons-dbcp:commons-dbcp:jar</exclude>
+ <exclude>commons-digester:commons-digester:jar</exclude>
+ <exclude>commons-io:commons-io:jar</exclude>
+ <exclude>commons-lang:commons-lang:jar</exclude>
+ <exclude>commons-net:commons-net:jar</exclude>
+ <exclude>commons-pool:commons-pool:jar</exclude>
+ <exclude>io.netty:netty:jar</exclude>
+ <exclude>io.netty:netty-all:jar</exclude>
+ <exclude>io.netty:netty-buffer:jar</exclude>
+ <exclude>io.netty:netty-codec:jar</exclude>
+ <exclude>io.netty:netty-codec-http:jar</exclude>
+ <exclude>io.netty:netty-common:jar</exclude>
+ <exclude>io.netty:netty-handler:jar</exclude>
+ <exclude>io.netty:netty-transport:jar</exclude>
+ <exclude>io.netty:netty-transport-native-epoll:jar</exclude>
+ <exclude>io.reactivex:rxjava:jar</exclude>
+ <exclude>io.reactivex:rxnetty:jar</exclude>
+ <exclude>io.reactivex:rxnetty-contexts:jar</exclude>
+ <exclude>io.reactivex:rxnetty-servo:jar</exclude>
+ <exclude>javax.activation:activation:jar</exclude>
+ <exclude>javax.annotation:javax.annotation-api:jar</exclude>
+ <exclude>javax.inject:javax.inject:jar</exclude>
+ <exclude>javax.servlet:javax.servlet-api:jar</exclude>
+ <exclude>javax.servlet.jsp:jsp-api:jar</exclude>
+ <exclude>javax.xml.bind:jaxb-api:jar</exclude>
+ <exclude>javax.xml.stream:stax-api:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude>
+ <exclude>org.antlr:antlr-runtime:jar</exclude>
+ <exclude>org.antlr:stringtemplate:jar</exclude>
+ <exclude>org.apache.commons:commons-compress:jar</exclude>
+ <exclude>org.apache.commons:commons-math3:jar</exclude>
+ <exclude>org.apache.curator:curator-framework:jar</exclude>
+ <exclude>org.apache.curator:curator-recipes:jar</exclude>
+ <exclude>org.apache.directory.api:api-asn1-api:jar</exclude>
+ <exclude>org.apache.directory.api:api-util:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-auth:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-common:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude>
+ <exclude>org.apache.htrace:htrace-core:jar</exclude>
+ <exclude>org.apache.logging.log4j:log4j-api:jar</exclude>
+ <exclude>org.apache.zookeeper:zookeeper:jar</exclude>
+ <exclude>org.aspectj:aspectjweaver:jar</exclude>
+ <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude>
+ <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude>
+ <exclude>org.codehaus.jettison:jettison:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-http:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-io:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-plus:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-security:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-server:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-util:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-xml:jar</exclude>
+ <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude>
+ <exclude>org.hdrhistogram:HdrHistogram:jar</exclude>
+ <exclude>org.json4s:json4s-ast_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-core_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-jackson_2.11:jar</exclude>
+ <exclude>org.jsoup:jsoup:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty-util:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude>
+ <exclude>org.scala-lang:scala-compiler:jar</exclude>
+ <exclude>org.scala-lang:scala-library:jar</exclude>
+ <exclude>org.scala-lang:scala-reflect:jar</exclude>
+ <exclude>org.scala-lang:scalap:jar</exclude>
+ <exclude>org.slf4j:jul-to-slf4j:jar</exclude>
+ <exclude>org.slf4j:slf4j-api:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.security:spring-security-crypto:jar</exclude>
+ <exclude>org.springframework.security:spring-security-rsa:jar</exclude>
+ <exclude>org.springframework:spring-aop:jar</exclude>
+ <exclude>org.springframework:spring-beans:jar</exclude>
+ <exclude>org.springframework:spring-context:jar</exclude>
+ <exclude>org.springframework:spring-core:jar</exclude>
+ <exclude>org.springframework:spring-expression:jar</exclude>
+ <!–<exclude>org.springframework:spring-jcl:jar</exclude>–>
+ <exclude>org.springframework:spring-web:jar</exclude>
+ <exclude>org.tukaani:xz:jar</exclude>
+ <exclude>org.yaml:snakeyaml:jar</exclude>
+ <exclude>xerces:xercesImpl:jar</exclude>
+ <exclude>xml-apis:xml-apis:jar</exclude>
+ <exclude>xmlenc:xmlenc:jar</exclude>
+ </excludes>-->
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/conf</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/bin</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>.</directory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <outputDirectory>logs</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
diff --git a/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveConnection.java b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveConnection.java
new file mode 100644
index 0000000..f581bdc
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveConnection.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+public class HiveConnection implements Closeable {
+
+ private Hive hiveClient;
+
+ private IMetaStoreClient metaStoreClient;
+
+ private static final CommonVars<String> KERBEROS_DEFAULT_PRINCIPLE =
+ CommonVars.apply("wds.linkis.server.mdm.service.kerberos.principle", "hadoop/_HOST@EXAMPLE.COM");
+
+ private static final CommonVars<String> DEFAULT_SERVICE_USER =
+ CommonVars.apply("wds.linkis.server.mdm.service.user", "hadoop");
+
+ private static final CommonVars<String> KERBEROS_KRB5_CONF_PATH =
+ CommonVars.apply("wds.linkis.server.mdm.service.kerberos.krb5.path", "");
+
+ static{
+ if(StringUtils.isNotBlank(KERBEROS_KRB5_CONF_PATH.getValue())){
+ System.setProperty("java.security.krb5.conf", KERBEROS_KRB5_CONF_PATH.getValue());
+ }
+ }
+
+ public HiveConnection(String uris, String principle, String keytabFilePath) throws Exception {
+ final HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.METASTOREURIS, uris);
+ conf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true");
+ conf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, KERBEROS_DEFAULT_PRINCIPLE.getValue());
+ //Disable the cache in FileSystem
+ conf.setBoolean(String.format("fs.%s.impl.disable.cache", URI.create(conf.get(FS_DEFAULT_NAME_KEY, "")).getScheme()), true);
+ conf.set("hadoop.security.authentication", "kerberos");
+ principle = principle.substring(0, principle.indexOf("@"));
+ UserGroupInformation ugi = UserGroupInformationWrapper.loginUserFromKeytab(conf,
+ principle, keytabFilePath);
+ hiveClient = getHive(ugi, conf);
+ }
+
+ public HiveConnection(String uris) throws Exception{
+ final HiveConf conf = new HiveConf();
+ conf.setVar(HiveConf.ConfVars.METASTOREURIS, uris);
+ //Disable the cache in FileSystem
+ conf.setBoolean(String.format("fs.%s.impl.disable.cache", URI.create(conf.get(FS_DEFAULT_NAME_KEY, "")).getScheme()), true);
+ //TODO choose an authentication strategy for hive, and then use createProxyUser
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(DEFAULT_SERVICE_USER.getValue());
+ hiveClient = getHive(ugi, conf);
+ }
+ /**
+ * Get Hive client(Hive object)
+ * @return hive
+ */
+ public Hive getClient(){
+ return hiveClient;
+ }
+
+ private Hive getHive(UserGroupInformation ugi, HiveConf conf) throws IOException, InterruptedException {
+ return ugi.doAs((PrivilegedExceptionAction<Hive>) () -> {
+ Hive hive = Hive.get(conf);
+ metaStoreClient = hive.getMSC();
+ //To remove thread Local vars
+ Hive.set(null);
+ return hive;
+ });
+ }
+ @Override
+ public void close() throws IOException {
+ //Close meta store client
+ metaStoreClient.close();
+ }
+
+ /**
+ * Wrapper class of UserGroupInformation
+ */
+ private static class UserGroupInformationWrapper{
+ private static ReentrantLock globalLock = new ReentrantLock();
+
+ public static UserGroupInformation loginUserFromKeytab(final Configuration conf, String user, String path) throws Exception {
+ globalLock.lock();
+ try{
+ UserGroupInformation.setConfiguration(conf);
+ return UserGroupInformation.loginUserFromKeytabAndReturnUGI(user, path);
+ }finally{
+ globalLock.unlock();
+ }
+ }
+ public static UserGroupInformation createProxyUser(final Configuration conf, String user) throws Exception{
+ globalLock.lock();
+ try{
+ UserGroupInformation.setLoginUser(null);
+ UserGroupInformation.setConfiguration(conf);
+ return UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
+ }finally{
+ globalLock.unlock();
+ }
+ }
+
+ public static UserGroupInformation getLoginUser() throws Exception{
+ globalLock.lock();
+ try{
+ UserGroupInformation.setLoginUser(null);
+ return UserGroupInformation.getLoginUser();
+ }finally{
+ globalLock.unlock();
+ }
+ }
+ }
+}
diff --git a/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveMetaService.java b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveMetaService.java
new file mode 100644
index 0000000..4ea3581
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveMetaService.java
@@ -0,0 +1,197 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.bml.client.BmlClient;
+import com.webank.wedatasphere.linkis.bml.client.BmlClientFactory;
+import com.webank.wedatasphere.linkis.bml.protocol.BmlDownloadResponse;
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaPartitionInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.exception.MetaRuntimeException;
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.AbstractMetaService;
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataConnection;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+@Component
+public class HiveMetaService extends AbstractMetaService<HiveConnection> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveMetaService.class);
+ private static final CommonVars<String> TMP_FILE_STORE_LOCATION =
+ CommonVars.apply("wds.linkis.server.mdm.service.temp.location", "classpath:/tmp");
+
+ private BmlClient client;
+
+ @PostConstruct
+ public void buildClient(){
+ client = BmlClientFactory.createBmlClient();
+ }
+ @Override
+ public MetadataConnection<HiveConnection> getConnection(String operator, Map<String, Object> params) throws Exception {
+ Resource resource = new PathMatchingResourcePatternResolver().getResource(TMP_FILE_STORE_LOCATION.getValue());
+ String uris = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_URIS.getValue(), ""));
+ String principle = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_PRINCIPLE.getValue(), ""));
+ HiveConnection conn = null;
+ if(StringUtils.isNotBlank(principle)){
+ LOG.info("Try to connect Hive MetaStore in kerberos with principle:[" + principle +"]");
+ String keytabResourceId = String.valueOf(params.getOrDefault(HiveParamsMapper.PARAM_HIVE_KEYTAB.getValue(), ""));
+ if(StringUtils.isNotBlank(keytabResourceId)){
+ LOG.info("Start to download resource id:[" + keytabResourceId +"]");
+ String keytabFilePath = resource.getFile().getAbsolutePath() + "/" + UUID.randomUUID().toString().replace("-", "");
+ if(!downloadResource(keytabResourceId, operator, keytabFilePath)){
+ throw new MetaRuntimeException("Fail to download resource i:[" + keytabResourceId +"]");
+ }
+ conn = new HiveConnection(uris, principle, keytabFilePath);
+ }else{
+ throw new MetaRuntimeException("Cannot find the keytab file in connect parameters");
+ }
+ }else{
+ conn = new HiveConnection(uris);
+ }
+ return new MetadataConnection<>(conn, true);
+ }
+
+ /**
+ * Download resource to path by id
+ * @param resourceId resource id
+ * @param user user
+ * @param absolutePath absolute path
+ * @return
+ * @throws IOException
+ */
+ private boolean downloadResource(String resourceId, String user, String absolutePath) throws IOException {
+ BmlDownloadResponse downloadResponse = client.downloadResource(user, resourceId, absolutePath);
+ if(downloadResponse.isSuccess()){
+ IOUtils.copy(downloadResponse.inputStream(), new FileOutputStream(absolutePath));
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public List<String> queryDatabases(HiveConnection connection) {
+ try {
+ return connection.getClient().getAllDatabases();
+ } catch (HiveException e) {
+ throw new RuntimeException("Fail to get Hive databases(获取数据库列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<String> queryTables(HiveConnection connection, String database) {
+ try {
+ return connection.getClient().getAllTables(database);
+ } catch (HiveException e) {
+ throw new RuntimeException("Fail to get Hive tables(获取表列表失败)", e);
+ }
+ }
+
+ @Override
+ public MetaPartitionInfo queryPartitions(HiveConnection connection, String database, String table) {
+ List<Partition> partitions;
+ Table rawTable;
+ try {
+ rawTable = connection.getClient().getTable(database, table);
+ partitions = connection.getClient().getPartitions(rawTable);
+ } catch (HiveException e) {
+ throw new RuntimeException("Fail to get Hive partitions(获取分区信息失败)", e);
+ }
+ MetaPartitionInfo info = new MetaPartitionInfo();
+ List<FieldSchema> partitionKeys = rawTable.getPartitionKeys();
+ List<String> partKeys = new ArrayList<>();
+ partitionKeys.forEach(e -> partKeys.add(e.getName()));
+ info.setPartKeys(partKeys);
+ //Static partitions
+ Map<String, MetaPartitionInfo.PartitionNode> pMap = new HashMap<>(20);
+ MetaPartitionInfo.PartitionNode root = new MetaPartitionInfo.PartitionNode();
+ info.setRoot(root);
+ long t = System.currentTimeMillis();
+ for(Partition p : partitions){
+ try {
+ List<String> values = p.getValues();
+ if(!partitionKeys.isEmpty()){
+ String parentNameValue = "";
+ for(int i = 0; i < values.size(); i++){
+ FieldSchema fieldSchema = partitionKeys.get(i);
+ String name = fieldSchema.getName();
+ String value = values.get(i);
+ String nameValue= name + "=" + value;
+ MetaPartitionInfo.PartitionNode node = new MetaPartitionInfo.PartitionNode();
+ if(i > 0){
+ MetaPartitionInfo.PartitionNode parent = pMap.get(parentNameValue);
+ parent.setName(name);
+ parent.getPartitions().putIfAbsent(value, node);
+ }else{
+ root.setName(name);
+ root.getPartitions().putIfAbsent(value, node);
+ }
+ parentNameValue += "/" + nameValue;
+ pMap.putIfAbsent(parentNameValue, node);
+ }
+ }
+ }catch(Exception e){
+ LOG.warn(e.getMessage(), e);
+ }
+ }
+ return info;
+ }
+
+ @Override
+ public List<MetaColumnInfo> queryColumns(HiveConnection connection, String database, String table) {
+ List<MetaColumnInfo> columns = new ArrayList<>();
+ Table tb;
+ try {
+ tb = connection.getClient().getTable(database, table);
+ } catch (HiveException e) {
+ throw new RuntimeException("Fail to get Hive columns(获得表字段信息失败)", e);
+ }
+ tb.getFields().forEach( field ->{
+ MetaColumnInfo metaColumnInfo = new MetaColumnInfo();
+ metaColumnInfo.setIndex(field.getFieldID());
+ metaColumnInfo.setName(field.getFieldName());
+ metaColumnInfo.setType(field.getFieldObjectInspector().getTypeName());
+ columns.add(metaColumnInfo);
+ });
+ return columns;
+ }
+
+ @Override
+ public Map<String, String> queryTableProps(HiveConnection connection, String database, String table) {
+ try {
+ Table rawTable = connection.getClient().getTable(database, table);
+ return new HashMap<>((Map)rawTable.getMetadata());
+ }catch(Exception e){
+ throw new RuntimeException("Fail to get Hive table properties(获取表参数信息失败)", e);
+ }
+ }
+}
diff --git a/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveParamsMapper.java b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveParamsMapper.java
new file mode 100644
index 0000000..6a3372b
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/HiveParamsMapper.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+public class HiveParamsMapper {
+
+ public static final CommonVars<String> PARAM_HIVE_PRINCIPLE =
+ CommonVars.apply("wds.linkis.server.mdm.service.hive.principle", "principle");
+
+ public static final CommonVars<String> PARAM_HIVE_URIS =
+ CommonVars.apply("wds.linkis.server.mdm.service.hive.uris", "uris");
+
+ public static final CommonVars<String> PARAM_HIVE_KEYTAB =
+ CommonVars.apply("wds.linkis.server.mdm.service.hive.keytab", "keytab");
+
+}
diff --git a/datasource/metadatamanager/service/hive/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/HiveReceiver.scala b/datasource/metadatamanager/service/hive/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/HiveReceiver.scala
new file mode 100644
index 0000000..9d13915
--- /dev/null
+++ b/datasource/metadatamanager/service/hive/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/HiveReceiver.scala
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service.receiver
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService
+import com.webank.wedatasphere.linkis.metadatamanager.common.receiver.BaseMetaReceiver
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService
+import javax.annotation.PostConstruct
+import org.springframework.stereotype.Component
+
+@Component
+class HiveReceiver extends BaseMetaReceiver{
+ @PostConstruct
+ def init(): Unit = {
+ metadataService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[MetadataService])
+ }
+}
diff --git a/datasource/metadatamanager/service/mysql/bin/start-mdm-mysql.sh b/datasource/metadatamanager/service/mysql/bin/start-mdm-mysql.sh
new file mode 100644
index 0000000..80cc775
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/bin/start-mdm-mysql.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+export SERVER_LOG_PATH=$HOME/logs
+export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication
+
+if test -z "$SERVER_HEAP_SIZE"
+then
+ export SERVER_HEAP_SIZE="512M"
+fi
+
+if test -z "$SERVER_JAVA_OPTS"
+then
+ export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log"
+fi
+
+if [[ -f "${SERVER_PID}" ]]; then
+ pid=$(cat ${SERVER_PID})
+ if kill -0 ${pid} >/dev/null 2>&1; then
+ echo "Server is already running."
+ exit 1
+ fi
+fi
+
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+pid=$!
+if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME start failed!"
+ exit 1
+else
+ echo "server $SERVER_NAME start succeeded!"
+ echo $pid > $SERVER_PID
+ sleep 1
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/mysql/bin/stop-mdm-mysql.sh b/datasource/metadatamanager/service/mysql/bin/stop-mdm-mysql.sh
new file mode 100644
index 0000000..f032887
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/bin/stop-mdm-mysql.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+cd `dirname $0`
+cd ..
+HOME=`pwd`
+
+export SERVER_PID=$HOME/bin/linkis.pid
+
+function wait_for_server_to_die() {
+ local pid
+ local count
+ pid=$1
+ timeout=$2
+ count=0
+ timeoutTime=$(date "+%s")
+ let "timeoutTime+=$timeout"
+ currentTime=$(date "+%s")
+ forceKill=1
+
+ while [[ $currentTime -lt $timeoutTime ]]; do
+ $(kill ${pid} > /dev/null 2> /dev/null)
+ if kill -0 ${pid} > /dev/null 2>&1; then
+ sleep 3
+ else
+ forceKill=0
+ break
+ fi
+ currentTime=$(date "+%s")
+ done
+
+ if [[ forceKill -ne 0 ]]; then
+ $(kill -9 ${pid} > /dev/null 2> /dev/null)
+ fi
+}
+
+if [[ ! -f "${SERVER_PID}" ]]; then
+ echo "server $SERVER_NAME is not running"
+else
+ pid=$(cat ${SERVER_PID})
+ if [[ -z "${pid}" ]]; then
+ echo "server $SERVER_NAME is not running"
+ else
+ wait_for_server_to_die $pid 40
+ $(rm -f ${SERVER_PID})
+ echo "server $SERVER_NAME is stopped."
+ fi
+fi
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/mysql/conf/application.yml b/datasource/metadatamanager/service/mysql/conf/application.yml
new file mode 100644
index 0000000..bb31831
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/conf/application.yml
@@ -0,0 +1,23 @@
+server:
+ port: 8294
+spring:
+ application:
+ name: mdm-service-mysql
+
+eureka:
+ client:
+ serviceUrl:
+ defaultZone: http://${ip}:${port}/eureka/
+ instance:
+ metadata-map:
+ test: wedatasphere
+
+management:
+ endpoints:
+ web:
+ exposure:
+ include: refresh,info
+logging:
+ config: classpath:log4j2.xml
+
+
diff --git a/datasource/metadatamanager/service/mysql/conf/linkis.properties b/datasource/metadatamanager/service/mysql/conf/linkis.properties
new file mode 100644
index 0000000..f83c10d
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/conf/linkis.properties
@@ -0,0 +1,21 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+wds.linkis.server.mybatis.mapperLocations=
+wds.linkis.server.mybatis.typeAliasesPackage=
+wds.linkis.server.mybatis.BasePackage=
+wds.linkis.server.restful.scan.packages=
+
+#sit
+wds.linkis.server.version=v1
+
diff --git a/datasource/metadatamanager/service/mysql/conf/log4j.properties b/datasource/metadatamanager/service/mysql/conf/log4j.properties
new file mode 100644
index 0000000..d5ee44b
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/conf/log4j.properties
@@ -0,0 +1,33 @@
+#
+# Copyright 2019 WeBank
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+### set log levels ###
+
+log4j.rootCategory=INFO,console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.Threshold=INFO
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n
+
+
+log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.com.webank.bdp.ide.core.Threshold=INFO
+log4j.additivity.com.webank.bdp.ide.core=false
+log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout
+log4j.appender.com.webank.bdp.ide.core.Append=true
+log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log
+log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n
+
+log4j.logger.org.springframework=INFO
\ No newline at end of file
diff --git a/datasource/metadatamanager/service/mysql/conf/log4j2.xml b/datasource/metadatamanager/service/mysql/conf/log4j2.xml
new file mode 100644
index 0000000..1c68190
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/conf/log4j2.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration status="error" monitorInterval="30">
+ <appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
+ </Console>
+ <RollingFile name="RollingFile" fileName="logs/linkis.log"
+ filePattern="logs/$${date:yyyy-MM}/linkis-log-%d{yyyy-MM-dd}-%i.log">
+ <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+ <SizeBasedTriggeringPolicy size="100MB"/>
+ <DefaultRolloverStrategy max="20"/>
+ </RollingFile>
+ </appenders>
+ <loggers>
+ <root level="INFO">
+ <appender-ref ref="RollingFile"/>
+ <appender-ref ref="Console"/>
+ </root>
+ </loggers>
+</configuration>
+
diff --git a/datasource/metadatamanager/service/mysql/pom.xml b/datasource/metadatamanager/service/mysql/pom.xml
new file mode 100644
index 0000000..1e671e4
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/pom.xml
@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>linkis</artifactId>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <version>0.9.4</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>linkis-metadatamanager-service-mysql</artifactId>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <mysql.version>5.1.34</mysql.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-metadatamanager-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-module</artifactId>
+ <version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>org.ow2.asm</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>${mysql.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ <inherited>false</inherited>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <skipAssembly>false</skipAssembly>
+ <finalName>linkis-mdm-service-mysql</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/main/assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/main/java</directory>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>**/*.properties</exclude>
+ <exclude>**/application.yml</exclude>
+ <exclude>**/bootstrap.yml</exclude>
+ <exclude>**/log4j2.xml</exclude>
+ </excludes>
+ </resource>
+ </resources>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </build>
+</project>
diff --git a/datasource/metadatamanager/service/mysql/src/main/assembly/distribution.xml b/datasource/metadatamanager/service/mysql/src/main/assembly/distribution.xml
new file mode 100644
index 0000000..75dbe65
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/src/main/assembly/distribution.xml
@@ -0,0 +1,206 @@
+<!--
+ ~ Copyright 2019 WeBank
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>linkis-mdm-service-mysql</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>linkis-mdm-service-mysql</baseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
+ <!-- Now, select which projects to include in this module-set. -->
+ <outputDirectory>lib</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <unpack>false</unpack>
+ <useStrictFiltering>true</useStrictFiltering>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+
+ <!-- <excludes>
+ <exclude>antlr:antlr:jar</exclude>
+ <exclude>aopalliance:aopalliance:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-annotations:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-core:jar</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-databind:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-paranamer:jar</exclude>
+ <exclude>com.fasterxml.jackson.module:jackson-module-scala_2.11:jar</exclude>
+ <exclude>com.google.code.gson:gson:jar</exclude>
+ <exclude>com.google.guava:guava:jar</exclude>
+ <exclude>com.google.protobuf:protobuf-java:jar</exclude>
+ <exclude>com.netflix.archaius:archaius-core:jar</exclude>
+ <exclude>com.netflix.hystrix:hystrix-core:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-commons-util:jar</exclude>
+ <exclude>com.netflix.netflix-commons:netflix-statistics:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-core:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-httpclient:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-loadbalancer:jar</exclude>
+ <exclude>com.netflix.ribbon:ribbon-transport:jar</exclude>
+ <exclude>com.netflix.servo:servo-core:jar</exclude>
+ <exclude>com.sun.jersey.contribs:jersey-apache-client4:jar</exclude>
+ <exclude>com.sun.jersey:jersey-client:jar</exclude>
+ <exclude>com.sun.jersey:jersey-core:jar</exclude>
+ <exclude>com.sun.jersey:jersey-server:jar</exclude>
+ <exclude>com.sun.xml.bind:jaxb-impl:jar</exclude>
+ <exclude>com.webank.wedatasphere.linkis:linkis-common:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils:jar</exclude>
+ <exclude>commons-beanutils:commons-beanutils-core:jar</exclude>
+ <exclude>commons-cli:commons-cli:jar</exclude>
+ <exclude>commons-collections:commons-collections:jar</exclude>
+ <exclude>commons-configuration:commons-configuration:jar</exclude>
+ <exclude>commons-daemon:commons-daemon:jar</exclude>
+ <exclude>commons-dbcp:commons-dbcp:jar</exclude>
+ <exclude>commons-digester:commons-digester:jar</exclude>
+ <exclude>commons-io:commons-io:jar</exclude>
+ <exclude>commons-lang:commons-lang:jar</exclude>
+ <exclude>commons-net:commons-net:jar</exclude>
+ <exclude>commons-pool:commons-pool:jar</exclude>
+ <exclude>io.netty:netty:jar</exclude>
+ <exclude>io.netty:netty-all:jar</exclude>
+ <exclude>io.netty:netty-buffer:jar</exclude>
+ <exclude>io.netty:netty-codec:jar</exclude>
+ <exclude>io.netty:netty-codec-http:jar</exclude>
+ <exclude>io.netty:netty-common:jar</exclude>
+ <exclude>io.netty:netty-handler:jar</exclude>
+ <exclude>io.netty:netty-transport:jar</exclude>
+ <exclude>io.netty:netty-transport-native-epoll:jar</exclude>
+ <exclude>io.reactivex:rxjava:jar</exclude>
+ <exclude>io.reactivex:rxnetty:jar</exclude>
+ <exclude>io.reactivex:rxnetty-contexts:jar</exclude>
+ <exclude>io.reactivex:rxnetty-servo:jar</exclude>
+ <exclude>javax.activation:activation:jar</exclude>
+ <exclude>javax.annotation:javax.annotation-api:jar</exclude>
+ <exclude>javax.inject:javax.inject:jar</exclude>
+ <exclude>javax.servlet:javax.servlet-api:jar</exclude>
+ <exclude>javax.servlet.jsp:jsp-api:jar</exclude>
+ <exclude>javax.xml.bind:jaxb-api:jar</exclude>
+ <exclude>javax.xml.stream:stax-api:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-core_2.11:jar</exclude>
+ <exclude>net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar</exclude>
+ <exclude>org.antlr:antlr-runtime:jar</exclude>
+ <exclude>org.antlr:stringtemplate:jar</exclude>
+ <exclude>org.apache.commons:commons-compress:jar</exclude>
+ <exclude>org.apache.commons:commons-math3:jar</exclude>
+ <exclude>org.apache.curator:curator-framework:jar</exclude>
+ <exclude>org.apache.curator:curator-recipes:jar</exclude>
+ <exclude>org.apache.directory.api:api-asn1-api:jar</exclude>
+ <exclude>org.apache.directory.api:api-util:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-i18n:jar</exclude>
+ <exclude>org.apache.directory.server:apacheds-kerberos-codec:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-annotations:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-auth:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-common:jar</exclude>
+ <exclude>org.apache.hadoop:hadoop-hdfs:jar</exclude>
+ <exclude>org.apache.htrace:htrace-core:jar</exclude>
+ <exclude>org.apache.logging.log4j:log4j-api:jar</exclude>
+ <exclude>org.apache.zookeeper:zookeeper:jar</exclude>
+ <exclude>org.aspectj:aspectjweaver:jar</exclude>
+ <exclude>org.bouncycastle:bcpkix-jdk15on:jar</exclude>
+ <exclude>org.bouncycastle:bcprov-jdk15on:jar</exclude>
+ <exclude>org.codehaus.jettison:jettison:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-continuation:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-http:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-io:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-jndi:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-plus:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-security:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-server:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-servlet:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-util:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-webapp:jar</exclude>
+ <exclude>org.eclipse.jetty:jetty-xml:jar</exclude>
+ <exclude>org.fusesource.leveldbjni:leveldbjni-all:jar</exclude>
+ <exclude>org.hdrhistogram:HdrHistogram:jar</exclude>
+ <exclude>org.json4s:json4s-ast_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-core_2.11:jar</exclude>
+ <exclude>org.json4s:json4s-jackson_2.11:jar</exclude>
+ <exclude>org.jsoup:jsoup:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty:jar</exclude>
+ <exclude>org.mortbay.jetty:jetty-util:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-parser-combinators_2.11:jar</exclude>
+ <exclude>org.scala-lang.modules:scala-xml_2.11:jar</exclude>
+ <exclude>org.scala-lang:scala-compiler:jar</exclude>
+ <exclude>org.scala-lang:scala-library:jar</exclude>
+ <exclude>org.scala-lang:scala-reflect:jar</exclude>
+ <exclude>org.scala-lang:scalap:jar</exclude>
+ <exclude>org.slf4j:jul-to-slf4j:jar</exclude>
+ <exclude>org.slf4j:slf4j-api:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-autoconfigure:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter:jar</exclude>
+ <exclude>org.springframework.boot:spring-boot-starter-aop:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-commons:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-context:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar</exclude>
+ <exclude>org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar</exclude>
+ <exclude>org.springframework.security:spring-security-crypto:jar</exclude>
+ <exclude>org.springframework.security:spring-security-rsa:jar</exclude>
+ <exclude>org.springframework:spring-aop:jar</exclude>
+ <exclude>org.springframework:spring-beans:jar</exclude>
+ <exclude>org.springframework:spring-context:jar</exclude>
+ <exclude>org.springframework:spring-core:jar</exclude>
+ <exclude>org.springframework:spring-expression:jar</exclude>
+ <!–<exclude>org.springframework:spring-jcl:jar</exclude>–>
+ <exclude>org.springframework:spring-web:jar</exclude>
+ <exclude>org.tukaani:xz:jar</exclude>
+ <exclude>org.yaml:snakeyaml:jar</exclude>
+ <exclude>xerces:xercesImpl:jar</exclude>
+ <exclude>xml-apis:xml-apis:jar</exclude>
+ <exclude>xmlenc:xmlenc:jar</exclude>
+ </excludes>-->
+ </dependencySet>
+ </dependencySets>
+
+ <fileSets>
+ <fileSet>
+ <directory>${basedir}/conf</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/bin</directory>
+ <includes>
+ <include>*</include>
+ </includes>
+ <fileMode>0777</fileMode>
+ <outputDirectory>bin</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ </fileSet>
+ <fileSet>
+ <directory>.</directory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ <outputDirectory>logs</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+</assembly>
+
diff --git a/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlConnection.java b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlConnection.java
new file mode 100644
index 0000000..383a0f1
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlConnection.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+public class SqlConnection implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SqlConnection.class);
+
+ private static final CommonVars<String> SQL_DRIVER_CLASS =
+ CommonVars.apply("wds.linkis.server.mdm.service.sql.driver", "com.mysql.jdbc.Driver");
+
+ private static final CommonVars<String> SQL_CONNECT_URL =
+ CommonVars.apply("wds.linkis.server.mdm.service.sql.url", "jdbc:mysql://%s:%s/%s");
+
+ private Connection conn;
+
+ private ConnectMessage connectMessage;
+
+ public SqlConnection(String host, Integer port,
+ String username, String password,
+ Map<String, Object> extraParams ) throws ClassNotFoundException, SQLException {
+ connectMessage = new ConnectMessage(host, port, username, password, extraParams);
+ conn = getDBConnection(connectMessage, "");
+ //Try to create statement
+ Statement statement = conn.createStatement();
+ statement.close();
+ }
+
+ public List<String> getAllDatabases() throws SQLException {
+ java.util.List<java.lang.String> dataBaseName = new ArrayList<>();
+ Statement stmt = null;
+ ResultSet rs = null;
+ try{
+ stmt = conn.createStatement();
+ rs = stmt.executeQuery("SHOW DATABASES");
+ while (rs.next()){
+ dataBaseName.add(rs.getString(1));
+ }
+ } finally {
+ closeResource(null, stmt, rs);
+ }
+ return dataBaseName;
+ }
+
+ public List<String> getAllTables(String database) throws SQLException {
+ List<String> tableNames = new ArrayList<>();
+ Statement stmt = null;
+ ResultSet rs = null;
+ try {
+ stmt = conn.createStatement();
+ rs = stmt.executeQuery("SHOW TABLES FROM `" + database + "`");
+ while (rs.next()) {
+ tableNames.add(rs.getString(1));
+ }
+ return tableNames;
+ } finally{
+ closeResource(null, stmt, rs);
+ }
+ }
+
+ public List<MetaColumnInfo> getColumns(String database, String table) throws SQLException, ClassNotFoundException {
+ List<MetaColumnInfo> columns = new ArrayList<>();
+ String columnSql = "SELECT * FROM `" + database +"`.`" + table + "` WHERE 1 = 2";
+ PreparedStatement ps = null;
+ ResultSet rs = null;
+ ResultSetMetaData meta = null;
+ try {
+ List<String> primaryKeys = getPrimaryKeys(getDBConnection(connectMessage, database), table);
+ ps = conn.prepareStatement(columnSql);
+ rs = ps.executeQuery();
+ meta = rs.getMetaData();
+ int columnCount = meta.getColumnCount();
+ for (int i = 1; i < columnCount + 1; i++) {
+ MetaColumnInfo info = new MetaColumnInfo();
+ info.setIndex(i);
+ info.setName(meta.getColumnName(i));
+ info.setType(meta.getColumnTypeName(i));
+ if(primaryKeys.contains(meta.getColumnName(i))){
+ info.setPrimaryKey(true);
+ }
+ columns.add(info);
+ }
+ }finally {
+ closeResource(null, ps, rs);
+ }
+ return columns;
+ }
+
+ /**
+ * Get primary keys
+ * @param connection connection
+ * @param table table name
+ * @return
+ * @throws SQLException
+ */
+ private List<String> getPrimaryKeys(Connection connection, String table) throws SQLException {
+ ResultSet rs = null;
+ List<String> primaryKeys = new ArrayList<>();
+ try {
+ DatabaseMetaData dbMeta = connection.getMetaData();
+ rs = dbMeta.getPrimaryKeys(null, null, table);
+ while(rs.next()){
+ primaryKeys.add(rs.getString("column_name"));
+ }
+ return primaryKeys;
+ }finally{
+ if(null != rs){
+ closeResource(connection, null, rs);
+ }
+ }
+ }
+
+ /**
+ * close database resource
+ * @param connection connection
+ * @param statement statement
+ * @param resultSet result set
+ */
+ private void closeResource(Connection connection, Statement statement, ResultSet resultSet){
+ try {
+ if(null != resultSet && !resultSet.isClosed()) {
+ resultSet.close();
+ }
+ if(null != statement && !statement.isClosed()){
+ statement.close();
+ }
+ if(null != connection && !connection.isClosed()){
+ connection.close();
+ }
+ }catch (SQLException e){
+ LOG.warn("Fail to release resource [" + e.getMessage() +"]", e);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ closeResource(conn, null, null);
+ }
+
+ /**
+ * @param connectMessage
+ * @param database
+ * @return
+ * @throws ClassNotFoundException
+ */
+ private Connection getDBConnection(ConnectMessage connectMessage, String database) throws ClassNotFoundException, SQLException {
+ String extraParamString = connectMessage.extraParams.entrySet()
+ .stream().map(e -> String.join("=", e.getKey(), String.valueOf(e.getValue())))
+ .collect(Collectors.joining("&"));
+ Class.forName(SQL_DRIVER_CLASS.getValue());
+ String url = String.format(SQL_CONNECT_URL.getValue(), connectMessage.host, connectMessage.port, database);
+ if(!connectMessage.extraParams.isEmpty()) {
+ url += "?" + extraParamString;
+ }
+ return DriverManager.getConnection(url, connectMessage.username, connectMessage.password);
+ }
+
+ /**
+ * Connect message
+ */
+ private static class ConnectMessage{
+ private String host;
+
+ private Integer port;
+
+ private String username;
+
+ private String password;
+
+ private Map<String, Object> extraParams;
+
+ public ConnectMessage(String host, Integer port,
+ String username, String password,
+ Map<String, Object> extraParams){
+ this.host = host;
+ this.port = port;
+ this.username = username;
+ this.password = password;
+ this.extraParams = extraParams;
+ }
+ }
+}
diff --git a/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlMetaService.java b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlMetaService.java
new file mode 100644
index 0000000..744ee4a
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlMetaService.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.metadatamanager.common.Json;
+import com.webank.wedatasphere.linkis.metadatamanager.common.domain.MetaColumnInfo;
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.AbstractMetaService;
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataConnection;
+import org.springframework.stereotype.Component;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+@Component
+public class SqlMetaService extends AbstractMetaService<SqlConnection> {
+ @Override
+ public MetadataConnection<SqlConnection> getConnection(String operator, Map<String, Object> params) throws Exception {
+ String host = String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_HOST.getValue(), ""));
+ //After deserialize, Integer will be Double, Why?
+ Integer port = (Double.valueOf(String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PORT.getValue(), 0)))).intValue();
+ String username = String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_USERNAME.getValue(), ""));
+ String password = String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PASSWORD.getValue(), ""));
+ Map<String, Object> extraParams = new HashMap<>();
+ Object sqlParamObj = params.get(SqlParamsMapper.PARAM_SQL_EXTRA_PARAMS.getValue());
+ if(null != sqlParamObj){
+ if(!(sqlParamObj instanceof Map)){
+ extraParams = Json.fromJson(String.valueOf(sqlParamObj), Map.class, String.class, Object.class);
+ }else{
+ extraParams = (Map<String, Object>)sqlParamObj;
+ }
+ }
+ assert extraParams != null;
+ return new MetadataConnection<>(new SqlConnection(host, port, username, password, extraParams));
+ }
+
+ @Override
+ public List<String> queryDatabases(SqlConnection connection) {
+ try {
+ return connection.getAllDatabases();
+ } catch (SQLException e) {
+ throw new RuntimeException("Fail to get Sql databases(获取数据库列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<String> queryTables(SqlConnection connection, String database) {
+ try {
+ return connection.getAllTables(database);
+ } catch (SQLException e) {
+ throw new RuntimeException("Fail to get Sql tables(获取表列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<MetaColumnInfo> queryColumns(SqlConnection connection, String database, String table) {
+ try {
+ return connection.getColumns(database, table);
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new RuntimeException("Fail to get Sql columns(获取字段列表失败)", e);
+ }
+ }
+}
diff --git a/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlParamsMapper.java b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlParamsMapper.java
new file mode 100644
index 0000000..c41285e
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/src/main/java/com/webank/wedatasphere/linkis/metadatamanager/service/SqlParamsMapper.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars;
+
+/**
+ * @author davidhua
+ * 2020/02/14
+ */
+public class SqlParamsMapper {
+
+ public static final CommonVars<String> PARAM_SQL_HOST =
+ CommonVars.apply("wds.linkis.server.mdm.service.sql.host", "host");
+
+ public static final CommonVars<String> PARAM_SQL_PORT =
+ CommonVars.apply("wds.linkis.server.mdm.service.sql.port", "port");
+
+ public static final CommonVars<String> PARAM_SQL_USERNAME =
+ CommonVars.apply("wds.linkis.server.mdm.service.sql.username", "username");
+
+ public static final CommonVars<String> PARAM_SQL_PASSWORD =
+ CommonVars.apply("wds.linkis.server.mdm.service.sql.password", "password");
+
+ public static final CommonVars<String> PARAM_SQL_EXTRA_PARAMS =
+ CommonVars.apply("wds.linkis.server.mdm.service.sql.params", "params");
+}
diff --git a/datasource/metadatamanager/service/mysql/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/SqlReceiver.scala b/datasource/metadatamanager/service/mysql/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/SqlReceiver.scala
new file mode 100644
index 0000000..11f24a9
--- /dev/null
+++ b/datasource/metadatamanager/service/mysql/src/main/scala/com/webank/wedatasphere/linkis/metadatamanager/service/receiver/SqlReceiver.scala
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.metadatamanager.service.receiver
+
+import com.webank.wedatasphere.linkis.DataWorkCloudApplication
+import com.webank.wedatasphere.linkis.metadatamanager.common.receiver.BaseMetaReceiver
+import com.webank.wedatasphere.linkis.metadatamanager.common.service.MetadataService
+import javax.annotation.PostConstruct
+import org.springframework.stereotype.Component
+
+@Component
+class SqlReceiver extends BaseMetaReceiver{
+ @PostConstruct
+ def init(): Unit = {
+ metadataService = DataWorkCloudApplication.getApplicationContext.getBean(classOf[MetadataService])
+ }
+}
diff --git a/db/linkis_ddl.sql b/db/linkis_ddl.sql
index 206e529..cf588c9 100644
--- a/db/linkis_ddl.sql
+++ b/db/linkis_ddl.sql
@@ -146,6 +146,7 @@
`is_hidden` tinyint(1) DEFAULT NULL COMMENT 'Whether it is hidden from user. If set to 1(true), then user cannot modify, however, it could still be used in back-end',
`is_advanced` tinyint(1) DEFAULT NULL COMMENT 'Whether it is an advanced parameter. If set to 1(true), parameters would be displayed only when user choose to do so',
`level` tinyint(1) DEFAULT NULL COMMENT 'Basis for displaying sorting in the front-end. Higher the level is, higher the rank the parameter gets',
+ `unit` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `application_id` (`application_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
@@ -175,7 +176,7 @@
`instance` varchar(50) DEFAULT NULL COMMENT 'An instance of Entrance, consists of IP address of the entrance server and port',
`exec_id` varchar(50) DEFAULT NULL COMMENT 'execution ID, consists of jobID(generated by scheduler), executeApplicationName , creator and instance',
`um_user` varchar(50) DEFAULT NULL COMMENT 'User name',
- `execution_code` text,
+ `execution_code` text COMMENT 'Run script. When exceeding 6000 lines, script would be stored in HDFS and its file path would be stored in database',
`progress` float DEFAULT NULL COMMENT 'Script execution progress, between zero and one',
`log_path` varchar(200) DEFAULT NULL COMMENT 'File path of the log files',
`result_location` varchar(200) DEFAULT NULL COMMENT 'File path of the result',
@@ -190,8 +191,11 @@
`script_path` varchar(200) DEFAULT NULL COMMENT 'Path of the script in workspace',
`params` text COMMENT 'Configuration item of the parameters',
`engine_instance` varchar(50) DEFAULT NULL COMMENT 'An instance of engine, consists of IP address of the engine server and port',
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+ `engine_start_time` time DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `created_time` (`created_time`),
+ KEY `um_user` (`um_user`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
DROP TABLE IF EXISTS `linkis_em_resource_meta_data`;
@@ -560,5 +564,155 @@
`end_time` datetime DEFAULT NULL COMMENT '结束时间',
`last_update_time` datetime NOT NULL COMMENT '最后更新时间',
PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+-- ----------------------------
+-- Table structure for linkis_cs_context_map
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_map`;
+CREATE TABLE `linkis_cs_context_map` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `key` varchar(128) DEFAULT NULL,
+ `context_scope` varchar(32) DEFAULT NULL,
+ `context_type` varchar(32) DEFAULT NULL,
+ `props` text,
+ `value` text,
+ `context_id` int(11) DEFAULT NULL,
+ `keywords` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `key` (`key`,`context_id`,`context_type`),
+ KEY `keywords` (`keywords`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_map_listener
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_map_listener`;
+CREATE TABLE `linkis_cs_context_map_listener` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `listener_source` varchar(255) DEFAULT NULL,
+ `key_id` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_history
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_history`;
+CREATE TABLE `linkis_cs_context_history` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `context_id` int(11) DEFAULT NULL,
+ `source` text,
+ `context_type` varchar(32) DEFAULT NULL,
+ `history_json` text,
+ `keyword` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `keyword` (`keyword`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_id
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_id`;
+CREATE TABLE `linkis_cs_context_id` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `user` varchar(32) DEFAULT NULL,
+ `application` varchar(32) DEFAULT NULL,
+ `source` varchar(255) DEFAULT NULL,
+ `expire_type` varchar(32) DEFAULT NULL,
+ `expire_time` datetime DEFAULT NULL,
+ `instance` varchar(32) DEFAULT NULL,
+ `backup_instance` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `instance` (`instance`),
+ KEY `backup_instance` (`backup_instance`(191)),
+ KEY `instance_2` (`instance`,`backup_instance`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_listener
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_listener`;
+CREATE TABLE `linkis_cs_context_listener` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `listener_source` varchar(255) DEFAULT NULL,
+ `context_id` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+
+
+-- ----------------------------
+-- Table structure for linkis_datasource
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS `linkis_datasource` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `datasource_name` VARCHAR(100) NOT NULL COMMENT 'Data source name',
+ `datasource_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Data source type id',
+ `datasource_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Data source description',
+ `create_identify` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Example: project name',
+ `create_system` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Create system',
+ `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator',
+ `parameter` TEXT COMMENT 'Connect parameters',
+ `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user',
+ `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time',
+ `datasource_env_id` BIGINT(20) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB AUTO_INCREMENT=140 DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Table structure for linkis_datasource_env
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS `linkis_datasource_env` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `env_name` VARCHAR(100) NOT NULL COMMENT 'Environment name',
+ `env_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Description',
+ `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator',
+ `parameter` TEXT NOT NULL COMMENT 'Connect parameters',
+ `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user',
+ `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time',
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB AUTO_INCREMENT=108 DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Table structure for linkis_datasource_type_key
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS `linkis_datasource_type_key` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `key` VARCHAR(50) DEFAULT NULL COMMENT 'Key of variable',
+ `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description',
+ `name` VARCHAR(50) DEFAULT NULL COMMENT 'Option name of column in page',
+ `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id',
+ `require` TINYINT(1) DEFAULT '0',
+ `scope` VARCHAR(50) DEFAULT NULL COMMENT 'Scope',
+ `default_value` VARCHAR(200) DEFAULT NULL COMMENT 'Default value',
+ `value_type` VARCHAR(50) DEFAULT NULL COMMENT 'Value type',
+ `value_regex` VARCHAR(100) DEFAULT NULL COMMENT 'Value regex',
+ `ref_id` BIGINT(20) DEFAULT NULL COMMENT 'Related id',
+ `ref_value` VARCHAR(100) DEFAULT NULL COMMENT 'Related value',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_datasource_type
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS `linkis_datasource_type` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `icon` VARCHAR(50) DEFAULT NULL COMMENT 'Icon',
+ `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description',
+ `name` VARCHAR(50) DEFAULT NULL COMMENT 'Name',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_datasource_type_env
+-- ----------------------------
+CREATE TABLE IF NOT EXISTS `linkis_datasource_type_env` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id',
+ `env_id` BIGINT(20) DEFAULT NULL COMMENT 'Environment id',
+ PRIMARY KEY (`id`)
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
\ No newline at end of file
diff --git a/db/linkis_dml.sql b/db/linkis_dml.sql
index fb000d0..cec6d54 100644
--- a/db/linkis_dml.sql
+++ b/db/linkis_dml.sql
@@ -38,8 +38,8 @@
INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.tmpfile.clean.time', 'tmp文件清理时间', 'tmp文件清理时间', @application_id, '10:00', 'None', NULL, '0', '0', '1');
INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.yarnqueue.cores.max', '取值范围：1-500，单位：个', '队列CPU使用上限', @application_id, '150', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|500)$', '0', '0', '1');
-INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.yarnqueue.memory.max', '取值范围：1-1000，单位：G', '队列内存使用上限', @application_id, '300G', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', '0', '0', '1');
-INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.client.memory.max', '取值范围：1-1000，单位：G', '驱动器内存使用上限', @application_id, '20G', 'Regex', '^([1-9]\\d{0,1}|100)(G|g)$', '0', '0', '1');
+INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `unit`) VALUES (0, 'wds.linkis.yarnqueue.memory.max', '取值范围：1-1000，单位：G', '队列内存使用上限', @application_id, '300G', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', '0', '0', '1', 'G');
+INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `unit`) VALUES (0, 'wds.linkis.client.memory.max', '取值范围：1-1000，单位：G', '驱动器内存使用上限', @application_id, '20G', 'Regex', '^([1-9]\\d{0,1}|100)(G|g)$', '0', '0', '1', 'G');
INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.instance', '范围：1-20，单位：个', '引擎最大并发数', @application_id, '10', 'NumInterval', '[1,20]', '0', '0', '1');
@@ -233,4 +233,8 @@
insert into `linkis_config_key_tree` VALUES(NULL,@key_id1,@tree_id1);
insert into `linkis_config_key_tree` VALUES(NULL,@key_id2,@tree_id1);
-insert into `linkis_config_key_tree` VALUES(NULL,@key_id3,@tree_id1);
\ No newline at end of file
+insert into `linkis_config_key_tree` VALUES(NULL,@key_id3,@tree_id1);
+
+INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'ElasticSearch');
+INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'Hive');
+INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'MySql');
\ No newline at end of file
diff --git a/db/module/linkis-cs.sql b/db/module/linkis-cs.sql
new file mode 100644
index 0000000..c69afde
--- /dev/null
+++ b/db/module/linkis-cs.sql
@@ -0,0 +1,81 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_map
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_map`;
+CREATE TABLE `linkis_cs_context_map` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `key` varchar(128) DEFAULT NULL,
+ `context_scope` varchar(32) DEFAULT NULL,
+ `context_type` varchar(32) DEFAULT NULL,
+ `props` text,
+ `value` text,
+ `context_id` int(11) DEFAULT NULL,
+ `keywords` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `key` (`key`,`context_id`,`context_type`),
+ KEY `keywords` (`keywords`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_map_listener
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_map_listener`;
+CREATE TABLE `linkis_cs_context_map_listener` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `listener_source` varchar(255) DEFAULT NULL,
+ `key_id` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_history
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_history`;
+CREATE TABLE `linkis_cs_context_history` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `context_id` int(11) DEFAULT NULL,
+ `source` text,
+ `context_type` varchar(32) DEFAULT NULL,
+ `history_json` text,
+ `keyword` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `keyword` (`keyword`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_id
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_id`;
+CREATE TABLE `linkis_cs_context_id` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `user` varchar(32) DEFAULT NULL,
+ `application` varchar(32) DEFAULT NULL,
+ `source` varchar(255) DEFAULT NULL,
+ `expire_type` varchar(32) DEFAULT NULL,
+ `expire_time` datetime DEFAULT NULL,
+ `instance` varchar(32) DEFAULT NULL,
+ `backup_instance` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `instance` (`instance`),
+ KEY `backup_instance` (`backup_instance`(191)),
+ KEY `instance_2` (`instance`,`backup_instance`(191))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_cs_context_listener
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_cs_context_listener`;
+CREATE TABLE `linkis_cs_context_listener` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `listener_source` varchar(255) DEFAULT NULL,
+ `context_id` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+
+
+
+
diff --git a/db/module/linkis-datasource.sql b/db/module/linkis-datasource.sql
new file mode 100644
index 0000000..40a7de9
--- /dev/null
+++ b/db/module/linkis-datasource.sql
@@ -0,0 +1,67 @@
+CREATE TABLE IF NOT EXISTS `linkis_datasource` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `datasource_name` VARCHAR(100) NOT NULL COMMENT 'Data source name',
+ `datasource_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Data source type id',
+ `datasource_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Data source description',
+ `create_identify` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Example: project name',
+ `create_system` VARCHAR(20) DEFAULT 'BDP' COMMENT 'Create system',
+ `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator',
+ `parameter` TEXT COMMENT 'Connect parameters',
+ `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user',
+ `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time',
+ `datasource_env_id` BIGINT(20) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB AUTO_INCREMENT=140 DEFAULT CHARSET=utf8;
+
+
+CREATE TABLE IF NOT EXISTS `linkis_datasource_env` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `env_name` VARCHAR(100) NOT NULL COMMENT 'Environment name',
+ `env_desc` VARCHAR(200) DEFAULT NULL COMMENT 'Description',
+ `create_user` VARCHAR(50) DEFAULT NULL COMMENT 'Creator',
+ `parameter` TEXT NOT NULL COMMENT 'Connect parameters',
+ `create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ `modify_user` VARCHAR(50) DEFAULT NULL COMMENT 'Modify user',
+ `modify_time` DATETIME DEFAULT NULL COMMENT 'Modify time',
+ PRIMARY KEY (`id`)
+ ) ENGINE=InnoDB AUTO_INCREMENT=108 DEFAULT CHARSET=utf8;
+
+
+CREATE TABLE IF NOT EXISTS `linkis_datasource_type_key` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `key` VARCHAR(50) DEFAULT NULL COMMENT 'Key of variable',
+ `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description',
+ `name` VARCHAR(50) DEFAULT NULL COMMENT 'Option name of column in page',
+ `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id',
+ `require` TINYINT(1) DEFAULT '0',
+ `scope` VARCHAR(50) DEFAULT NULL COMMENT 'Scope',
+ `default_value` VARCHAR(200) DEFAULT NULL COMMENT 'Default value',
+ `value_type` VARCHAR(50) DEFAULT NULL COMMENT 'Value type',
+ `value_regex` VARCHAR(100) DEFAULT NULL COMMENT 'Value regex',
+ `ref_id` BIGINT(20) DEFAULT NULL COMMENT 'Related id',
+ `ref_value` VARCHAR(100) DEFAULT NULL COMMENT 'Related value',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+CREATE TABLE IF NOT EXISTS `linkis_datasource_type` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `icon` VARCHAR(50) DEFAULT NULL COMMENT 'Icon',
+ `description` VARCHAR(200) DEFAULT NULL COMMENT 'Description',
+ `name` VARCHAR(50) DEFAULT NULL COMMENT 'Name',
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+CREATE TABLE IF NOT EXISTS `linkis_datasource_type_env` (
+ `id` BIGINT(20) NOT NULL AUTO_INCREMENT,
+ `data_source_type_id` BIGINT(20) DEFAULT NULL COMMENT 'Type id',
+ `env_id` BIGINT(20) DEFAULT NULL COMMENT 'Environment id',
+ PRIMARY KEY (`id`)
+)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'ElasticSearch');
+INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'Hive');
+INSERT INTO `linkis_datasource_type`(`icon`, `name`) VALUES('0x001', 'MySql');
\ No newline at end of file
diff --git a/db/module/linkis_configuration.sql b/db/module/linkis_configuration.sql
index acc92cd..1f43322 100644
--- a/db/module/linkis_configuration.sql
+++ b/db/module/linkis_configuration.sql
@@ -61,6 +61,7 @@
`is_hidden` tinyint(1) DEFAULT NULL COMMENT 'Whether it is hidden from user. If set to 1(true), then user cannot modify, however, it could still be used in back-end',
`is_advanced` tinyint(1) DEFAULT NULL COMMENT 'Whether it is an advanced parameter. If set to 1(true), parameters would be displayed only when user choose to do so',
`level` tinyint(1) DEFAULT NULL COMMENT 'Basis for displaying sorting in the front-end. Higher the level is, higher the rank the parameter gets',
+ `unit` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `application_id` (`application_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
diff --git a/db/module/linkis_configuration_dml.sql b/db/module/linkis_configuration_dml.sql
index 7b3b8e7..4e845ac 100644
--- a/db/module/linkis_configuration_dml.sql
+++ b/db/module/linkis_configuration_dml.sql
@@ -43,8 +43,8 @@
#INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.tmpfile.clean.time', 'tmp文件清理时间', 'tmp文件清理时间', @application_id, '10:00', 'None', NULL, '0', '0', '1');
#INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.yarnqueue.cores.max', '取值范围：1-500，单位：个', '队列CPU使用上限', @application_id, '150', 'Regex', '^(?:[1-9]\\d?|[1234]\\d{2}|500)$', '0', '0', '1');
-#INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.yarnqueue.memory.max', '取值范围：1-1000，单位：G', '队列内存使用上限', @application_id, '300G', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', '0', '0', '1');
-INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.client.memory.max', '取值范围：1-1000，单位：G', '驱动器内存使用上限', @application_id, '20G', 'Regex', '^([1-9]\\d{0,1}|100)(G|g)$', '0', '0', '1');
+#INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `unit`) VALUES (0, 'wds.linkis.yarnqueue.memory.max', '取值范围：1-1000，单位：G', '队列内存使用上限', @application_id, '300G', 'Regex', '^([1-9]\\d{0,2}|1000)(G|g)$', '0', '0', '1','G');
+INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `unit`) VALUES (0, 'wds.linkis.client.memory.max', '取值范围：1-100，单位：G', '驱动器内存使用上限', @application_id, '20G', 'Regex', '^([1-9]\\d{0,1}|100)(G|g)$', '0', '0', '1', 'G');
#INSERT INTO `linkis_config_key` (`id`, `key`, `description`, `name`, `application_id`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`) VALUES (0, 'wds.linkis.instance', '范围：1-20，单位：个', '引擎最大并发数', @application_id, '10', 'NumInterval', '[1,20]', '0', '0', '1');
diff --git a/db/module/linkis_query.sql b/db/module/linkis_query.sql
index aaef124..1262fba 100644
--- a/db/module/linkis_query.sql
+++ b/db/module/linkis_query.sql
@@ -24,7 +24,10 @@
`script_path` varchar(200) DEFAULT NULL COMMENT 'Path of the script in workspace',
`params` text COMMENT 'Configuration item of the parameters',
`engine_instance` varchar(50) DEFAULT NULL COMMENT 'An instance of engine, consists of IP address of the engine server and port',
- PRIMARY KEY (`id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+ `engine_start_time` time DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `created_time` (`created_time`),
+ KEY `um_user` (`um_user`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
diff --git a/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md b/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md
index d7b679a..ed67b02 100644
--- a/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md
+++ b/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md
@@ -1,86 +1,86 @@
-# How to adapt Linkis with a new computation or storage engine
-
-## 1. Introduction
-
-Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements.
-Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis.
-The purpose and archetecture of these three modules please refer to Linkis Archetect Design Docs[UJES架构设计文档](../ch4/Linkis-UJES设计文档.md).
-
-
-## 2. Declaration
-
-Linkis uses Spring framework as the underlying technique. So the Spring development specs must be obeyed.
-
-Linkis has an elastic underlying achitecture and provides common implementations for almost all of its top-level interfaces. If customized classes are needed by users, they can be directly injected and replace the current implementations.
-
-
-### 2.1 Entrance module adaption
-
-**1) maven dependency**
-
-```xml
-<dependency>
- <groupId>com.webank.wedatasphere.Linkis</groupId>
- <artifactId>Linkis-ujes-entrance</artifactId>
- <version>0.9.3</version>
-</dependency>
-```
-
-**2)Interfaces to be implemented**
-
-There is no compulsary interface in Entrance. Below interfaces can be implemented on demand.
-- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overrided. Linkis provides CommonEntranceParser as the default implementation.
-- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager.
-- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes.
-
-### 2.2 EngineManager module adaption
-
-**1) maven dependency**
-
-```xml
-<dependency>
- <groupId>com.webank.wedatasphere.Linkis</groupId>
- <artifactId>Linkis-ujes-enginemanager</artifactId>
- <version>0.9.3</version>
-</dependency>
-```
-
-**2)Interfaces to be implemented**
-
-Below interfaces are required to be implemented in EngineManager:
-- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder.
-Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified.
-- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements.
-- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection.
-
-Below interfaces/beans are optional in EngineManager:
-- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to declare an Spring bean in type EngineHook[] hooks to make new hooks effective. For details please refer to com.webank.wedatasphere.linkis.enginemanager.impl.EngineManagerSpringConfiguration.
-
-
-### 2.3 Engine module adaption
-
-**1) maven dependency**
-
-```xml
-<dependency>
- <groupId>com.webank.wedatasphere.Linkis</groupId>
- <artifactId>Linkis-ujes-engine</artifactId>
- <version>0.9.3</version>
-</dependency>
-```
-
-
-**2)Interfaces to be implemented**
-
-Below interfaces are required to be implemented in Engine：
-- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. This map contains evironment variables and engine arguments.
-- EngineExecutor. The actual executor to execute the code submitted from the entrance.
- Methods need to be implemented:
- 1. getActualUsedResources(the resource an engine acually used)
- 2. executeLine(execute a line of the code parsed by CodeParser)
- 3. executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine)
-
-Below interfaces/beans are optional in Engine:
-- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default.
-- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default.
-- EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob.
+# How to adapt Linkis with a new computation or storage engine
+
+## 1. Introduction
+
+Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements.
+Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis.
+The purpose and archetecture of these three modules please refer to Linkis Archetect Design Docs[UJES架构设计文档](../ch4/Linkis-UJES设计文档.md).
+
+
+## 2. Declaration
+
+Linkis uses Spring framework as the underlying technique. So the Spring development specs must be obeyed.
+
+Linkis has an elastic underlying achitecture and provides common implementations for almost all of its top-level interfaces. If customized classes are needed by users, they can be directly injected and replace the current implementations.
+
+
+### 2.1 Entrance module adaption
+
+**1) maven dependency**
+
+```xml
+<dependency>
+ <groupId>com.webank.wedatasphere.Linkis</groupId>
+ <artifactId>Linkis-ujes-entrance</artifactId>
+ <version>0.9.4</version>
+</dependency>
+```
+
+**2)Interfaces to be implemented**
+
+There is no compulsary interface in Entrance. Below interfaces can be implemented on demand.
+- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overrided. Linkis provides CommonEntranceParser as the default implementation.
+- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager.
+- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes.
+
+### 2.2 EngineManager module adaption
+
+**1) maven dependency**
+
+```xml
+<dependency>
+ <groupId>com.webank.wedatasphere.Linkis</groupId>
+ <artifactId>Linkis-ujes-enginemanager</artifactId>
+ <version>0.9.4</version>
+</dependency>
+```
+
+**2)Interfaces to be implemented**
+
+Below interfaces are required to be implemented in EngineManager:
+- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder.
+Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified.
+- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements.
+- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection.
+
+Below interfaces/beans are optional in EngineManager:
+- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to declare an Spring bean in type EngineHook[] hooks to make new hooks effective. For details please refer to com.webank.wedatasphere.linkis.enginemanager.impl.EngineManagerSpringConfiguration.
+
+
+### 2.3 Engine module adaption
+
+**1) maven dependency**
+
+```xml
+<dependency>
+ <groupId>com.webank.wedatasphere.Linkis</groupId>
+ <artifactId>Linkis-ujes-engine</artifactId>
+ <version>0.9.4</version>
+</dependency>
+```
+
+
+**2)Interfaces to be implemented**
+
+Below interfaces are required to be implemented in Engine：
+- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. This map contains evironment variables and engine arguments.
+- EngineExecutor. The actual executor to execute the code submitted from the entrance.
+ Methods need to be implemented:
+ 1. getActualUsedResources(the resource an engine acually used)
+ 2. executeLine(execute a line of the code parsed by CodeParser)
+ 3. executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine)
+
+Below interfaces/beans are optional in Engine:
+- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default.
+- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default.
+- EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob.
diff --git a/docs/en_US/ch3/Linkis User Manual.md b/docs/en_US/ch3/Linkis User Manual.md
index 6ab6c18..c5fac81 100644
--- a/docs/en_US/ch3/Linkis User Manual.md
+++ b/docs/en_US/ch3/Linkis User Manual.md
@@ -1,356 +1,356 @@
-## Linkis User Manual
-
-#### 1.Summary
- Designed by Webank independently, Linkis is an extensible framework and a sophisticated solution for big data task submission. Conveniently, it could be used directly together with Scriptest, which is another open-source project powered by Webank. And those frontend APIs are also available for users. A client implementation is also provided as an SDK to interact directly with background services. As an highly extensible framework, users can leverage the SDK to develop their own applications.
-
-#### 2.Frontend Adaption
- Two protocols are supported for frontend APIs: HTTP and WebSocket. Compared with HTTP, Websocket is more friendly to servers and behaved more efficiently in message pushing. But WebSocket is unstable and users are easily to be disconnected by accident. So Scriptest combined both ways to adapt with Linkis. It communicates with Linkis by websocket in normal circumstances, and failover to HTTP protocol in case the Websocket connection was down.
-##### 2.1 API Specs
-Linkis has its own specs for front-backend adaption.<br>
-
-**1).URL specs**
-```
-/api/rest_j/v1/{applicationName}/.+
-/api/rest_s/v1/{applicationName}/.+
-```
-
-- rest_j means the API is conformed to Jersey standards
-- rest_s means the API is conformed to springMVC Rest standards
-- v1 is the version of services，**The version will be upgraded with Linkis releases**
-- {applicationName} is the microservice name
-
-**2).Request specs**
-```json
-{
- "method":"/api/rest_j/v1/entrance/execute",
- "data":{},
- "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
-}
-```
-
-**3).Response specs**
-```json
-{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":Success！","data":{}}
-```
-- method：Return the Restful API URL requested, basically used by websocket protocol。
-- status：Return the status info, in which -1 means login failed, 0 means succeeded, 1 means error, 2 mean validation failed, and 3 means no permission.
-- data：Return detailed data.
-- message：Return hint message of the request. If the status is not 0, this message returns error messages. At the same time 'data' may return the stack information in its 'stack' column.
-
-
-##### 2.2WebSocket API Description
-
-
-**1).Establish connection**<br>
-<br>
-Used to establish a WebSocket connection with Linkis.
-- API `/api/rest_j/entrance/connect`
-- HTTP Method **GET**
-- Status Code **101**<br>
-
-**2).Request execution**<br>
-<br>
-Used to submit user jobs to Linkis for execution.
-- API `/api/rest_j/entrance/execute`
-- HTTP Method `POST`<br>
-- Sample Json request body
-```json
-{
- "method":"/api/rest_j/v1/entrance/execute",
- "data":{
- "params": {
- "variable":{
- "k1":"v1"
- },
- "configuration":{
- "special":{
- "k2":"v2"
- },
- "runtime":{
- "k3":"v3"
- },
- "startup":{
- "k4":"v4"
- }
- }
- },
- "executeApplicationName":"spark",
- "executionCode":"show tables",
- "runType":"sql",
- "source":{
- "scriptPath": "/home//linkis.sql"
- },
- "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
- }
-}
-```
-- Descriptions for parameters of request body data<br>
-
-| Parameter Name | Parameter Definition | Type | Comments |
-| ------------ | ------------ | ------------ | ------------ |
-| executeApplicationName | The Engine service expected by the user, such as Spark or hive| String | Not null |
-| requestApplicationName | The name of the system launching this request | String | Nullable |
-| params | User-defined parameters to run services | Map | Required, but values are nullable |
-| executionCode | The execution code submitted by the user | String |Not null |
-| runType | Assuming that the user executes a spark job, he may choose python, R or SQL as runType| String | Not null |
-| scriptPath | The script path of the execution code | String | For Scriptest, it shouldn't be null with executionCode at the same time |
- Table 1 Descriptions for the parameters
-
-
-- Sample Json response body
-```json
-{
- "method": "/api/rest_j/v1/entrance/execute",
- "status": 0,
- "message": "Execution request succeeded",
- "data": {
- "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21",
- "taskID": "123"
- }
-}
-```
-- execID is a unique ID of String type generated for each user task after submitted to Linkis. It is only used during the execution period, like PID. The format of execID is (length of requestApplicationName)(length of executeAppName)(length of Instance)${requestApplicationName}${executeApplicationName}${entranceInstance infomation ip+port}${requestApplicationName}_${umUser}_${index}
-- taskID is a unique ID of Long type genenrated incrementally by the database for each task.
-
-**3).The push mechanism for task status, logs and progress**<br>
-
-After submission, the status, logs and progress information will be pushed by the server. They could be retrieved by websocket protocol.
-The API is consistent with HTTP protocol as mentioned below. The only difference is that the schema of websocket is ws://, but http:// for HTTP protocol.
-
-Sample response of WebSocket API
-- Logs
-```json
-{
- "method": "/api/rest_j/v1/entrance/${execID}/log",
- "status": 0,
- "message": "Returned log information",
- "data": {
- "execID": "${execID}",
- "log": ["errorLog","warnLog","infoLog", "allLog"],
- "taskID":28594,
- "fromLine": 56
- },
- "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
-}
-```
-- Status
-```json
-{
- "method": "/api/rest_j/v1/entrance/${execID}/status",
- "status": 0,
- "message": "Return status information",
- "data": {
- "execID": "${execID}",
- "taskID":28594,
- "status": "Running",
- },
- "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
-}
-```
-- Progress
-```json
-{
- "method": "/api/rest_j/v1/entrance/${execID}/log",
- "status": 0,
- "message": "Return progress information",
- "data": {
- "execID": "${execID}",
- "taskID":28594,
- "progress": 0.2,
- "progressInfo": [
- {
- "id": "job-1",
- "succeedTasks": 2,
- "failedTasks": 0,
- "runningTasks": 5,
- "totalTasks": 10
- },
- {
- "id": "job-2",
- "succeedTasks": 5,
- "failedTasks": 0,
- "runningTasks": 5,
- "totalTasks": 10
- }
- ]
- },
- "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
-}
-```
-
-
-
-##### 2.3HTTP API description
-For HTTP API, polling should be used to retrieve the status, logs and progress information after submission.
-
-
-**1).Request execution**
-
-- API `/api/rest_j/entrance/execute`
-- HTTP Method `POST`<br>
-- Sample JSON request body, its description is the same with Table 1
-```json
-{
- "params": {},
- "executeApplicationName":"spark",
- "executionCode":"show tables",
- "runType":"sql",
- "source":{
- "scriptPath": "/home/linkis/linkis.sql"
- }
-}
-```
-- The response is consistent with websocket. Both execId and taskId will be obtained.
-
-**2).Retrieve status**<br>
-<br>
-- API `/api/rest_j/entrance/${execID}/status`
-- HTTP Method `GET`<br>
-- Sample response body
-```json
-{
- "method": "/api/rest_j/v1/entrance/{execID}/status",
- "status": 0,
- "message": "Succeeded to retrieve status",
- "data": {
- "execID": "${execID}",
- "status": "Running"
- }
-}
-```
-
-**3).Retrieve logs**<br>
-<br>
-- API `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}`
-- HTTP Method `GET`
-- Parameter fromLine specifies from which line to start. Parameter size specifies the number of lines should be retrieved for this request.
-- Sample response body, the returned fromLine indicates the value of parameter fromLine for next request.
-```json
-{
- "method": "/api/rest_j/v1/entrance/${execID}/log",
- "status": 0,
- "message": "Return logs information",
- "data": {
- "execID": "${execID}",
- "log": ["errorLogs","warnLogs","infoLogs", "allLogs],
- "fromLine": 56
- }
-}
-```
-
-**4).Retrieve progress**<br>
-<br>
-- API `/api/rest_j/entrance/${execID}/progress`
-- HTTP Method `GET`<br>
-- Sample response body
-```json
-{
- "method": "/api/rest_j/v1/entrance/{execID}/progress",
- "status": 0,
- "message": "Return progress information",
- "data": {
- "execID": "${execID}",
- "progress": 0.2,
- "progressInfo": [
- {
- "id": "job-1",
- "succeedTasks": 2,
- "failedTasks": 0,
- "runningTasks": 5,
- "totalTasks": 10
- },
- {
- "id": "job-2",
- "succeedTasks": 5,
- "failedTasks": 0,
- "runningTasks": 5,
- "totalTasks": 10
- }
- ]
- }
-}
-```
-**5).kill task**<br>
-<br>
-- API `/api/rest_j/entrance/${execID}/kill`
-- HTTP Method `POST`
-- Sample response body
-```json
-{
- "method": "/api/rest_j/v1/entrance/{execID}/kill",
- "status": 0,
- "message": "OK",
- "data": {
- "execID":"${execID}"
- }
-}
-```
-
-
-
-### 3.Client SDK Adaption
-[Please see Linkis Quick Start](/docs/en_US/ch2/Linkis%20Quick%20Start.md)
-
-
-
-#### 4. Multiple engine type support
-Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. The purpose and architecture of these three modules please refer to Linkis Architect Design Docs.
-
-#### Convention
-
-Linkis uses Spring framework as the underlying technique. So instances of some classes can be injected using Spring annotations. Linkis provides some default common implementations. If customized classes are needed by users, they can be directly injected and replace the current implementations.
-
-##### 4.1Entrance module adaption
-**1)maven dependency**
-```xml
-<dependency>
- <groupId>com.webank.wedatasphere.linkis</groupId>
- <artifactId>linkis-ujes-entrance</artifactId>
- <version>0.9.3</version>
-</dependency>
-```
-**2)Interfaces to be implemented**
-
-There is no compulsory interface in Entrance. Below interfaces can be implemented on demand.
-- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overridden. Linkis provides CommonEntranceParser as the default implementation.
-- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager.
-- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes.
-
-##### 4.2EngineManager module adaption
-**1)maven dependency**
-```xml
-<dependency>
- <groupId>com.webank.wedatasphere.linkis</groupId>
- <artifactId>linkis-ujes-enginemanager</artifactId>
- <version>0.9.3</version>
-</dependency>
-```
-
-**2)Interfaces to be implemented**
-
-Below interfaces are required to be implemented in EngineManager:
-- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder.
-Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified.
-- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements.
-- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to specify an Array[EngineHook] for dependency injection.
-- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection.
-
-##### 4.3Engine module adaption
-**1)maven dependency**
-```xml
-<dependency>
- <groupId>com.webank.wedatasphere.linkis</groupId>
- <artifactId>linkis-ujes-engine</artifactId>
- <version>0.9.3</version>
-</dependency>
-```
-**2)Interfaces to be implemented**
-
-Below interfaces are required to be implemented in Engine：
-- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor.
-- EngineExecutor. The actual executor to execute the code submitted from the entrance. Methods need to be implemented: getActualUsedResources(the resource an engine acually used), executeLine(execute a line of the code parsed by CodeParser), executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine)
-
-Below interfaces/beans are optional in Engine:
-- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default.
-- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default.
+## Linkis User Manual
+
+#### 1.Summary
+ Designed by Webank independently, Linkis is an extensible framework and a sophisticated solution for big data task submission. Conveniently, it could be used directly together with Scriptest, which is another open-source project powered by Webank. And those frontend APIs are also available for users. A client implementation is also provided as an SDK to interact directly with background services. As an highly extensible framework, users can leverage the SDK to develop their own applications.
+
+#### 2.Frontend Adaption
+ Two protocols are supported for frontend APIs: HTTP and WebSocket. Compared with HTTP, Websocket is more friendly to servers and behaved more efficiently in message pushing. But WebSocket is unstable and users are easily to be disconnected by accident. So Scriptest combined both ways to adapt with Linkis. It communicates with Linkis by websocket in normal circumstances, and failover to HTTP protocol in case the Websocket connection was down.
+##### 2.1 API Specs
+Linkis has its own specs for front-backend adaption.<br>
+
+**1).URL specs**
+```
+/api/rest_j/v1/{applicationName}/.+
+/api/rest_s/v1/{applicationName}/.+
+```
+
+- rest_j means the API is conformed to Jersey standards
+- rest_s means the API is conformed to springMVC Rest standards
+- v1 is the version of services，**The version will be upgraded with Linkis releases**
+- {applicationName} is the microservice name
+
+**2).Request specs**
+```json
+{
+ "method":"/api/rest_j/v1/entrance/execute",
+ "data":{},
+ "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
+}
+```
+
+**3).Response specs**
+```json
+{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":Success！","data":{}}
+```
+- method：Return the Restful API URL requested, basically used by websocket protocol。
+- status：Return the status info, in which -1 means login failed, 0 means succeeded, 1 means error, 2 mean validation failed, and 3 means no permission.
+- data：Return detailed data.
+- message：Return hint message of the request. If the status is not 0, this message returns error messages. At the same time 'data' may return the stack information in its 'stack' column.
+
+
+##### 2.2WebSocket API Description
+
+
+**1).Establish connection**<br>
+<br>
+Used to establish a WebSocket connection with Linkis.
+- API `/api/rest_j/entrance/connect`
+- HTTP Method **GET**
+- Status Code **101**<br>
+
+**2).Request execution**<br>
+<br>
+Used to submit user jobs to Linkis for execution.
+- API `/api/rest_j/entrance/execute`
+- HTTP Method `POST`<br>
+- Sample Json request body
+```json
+{
+ "method":"/api/rest_j/v1/entrance/execute",
+ "data":{
+ "params": {
+ "variable":{
+ "k1":"v1"
+ },
+ "configuration":{
+ "special":{
+ "k2":"v2"
+ },
+ "runtime":{
+ "k3":"v3"
+ },
+ "startup":{
+ "k4":"v4"
+ }
+ }
+ },
+ "executeApplicationName":"spark",
+ "executionCode":"show tables",
+ "runType":"sql",
+ "source":{
+ "scriptPath": "/home//linkis.sql"
+ },
+ "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
+ }
+}
+```
+- Descriptions for parameters of request body data<br>
+
+| Parameter Name | Parameter Definition | Type | Comments |
+| ------------ | ------------ | ------------ | ------------ |
+| executeApplicationName | The Engine service expected by the user, such as Spark or hive| String | Not null |
+| requestApplicationName | The name of the system launching this request | String | Nullable |
+| params | User-defined parameters to run services | Map | Required, but values are nullable |
+| executionCode | The execution code submitted by the user | String |Not null |
+| runType | Assuming that the user executes a spark job, he may choose python, R or SQL as runType| String | Not null |
+| scriptPath | The script path of the execution code | String | For Scriptest, it shouldn't be null with executionCode at the same time |
+ Table 1 Descriptions for the parameters
+
+
+- Sample Json response body
+```json
+{
+ "method": "/api/rest_j/v1/entrance/execute",
+ "status": 0,
+ "message": "Execution request succeeded",
+ "data": {
+ "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21",
+ "taskID": "123"
+ }
+}
+```
+- execID is a unique ID of String type generated for each user task after submitted to Linkis. It is only used during the execution period, like PID. The format of execID is (length of requestApplicationName)(length of executeAppName)(length of Instance)${requestApplicationName}${executeApplicationName}${entranceInstance infomation ip+port}${requestApplicationName}_${umUser}_${index}
+- taskID is a unique ID of Long type genenrated incrementally by the database for each task.
+
+**3).The push mechanism for task status, logs and progress**<br>
+
+After submission, the status, logs and progress information will be pushed by the server. They could be retrieved by websocket protocol.
+The API is consistent with HTTP protocol as mentioned below. The only difference is that the schema of websocket is ws://, but http:// for HTTP protocol.
+
+Sample response of WebSocket API
+- Logs
+```json
+{
+ "method": "/api/rest_j/v1/entrance/${execID}/log",
+ "status": 0,
+ "message": "Returned log information",
+ "data": {
+ "execID": "${execID}",
+ "log": ["errorLog","warnLog","infoLog", "allLog"],
+ "taskID":28594,
+ "fromLine": 56
+ },
+ "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
+}
+```
+- Status
+```json
+{
+ "method": "/api/rest_j/v1/entrance/${execID}/status",
+ "status": 0,
+ "message": "Return status information",
+ "data": {
+ "execID": "${execID}",
+ "taskID":28594,
+ "status": "Running",
+ },
+ "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
+}
+```
+- Progress
+```json
+{
+ "method": "/api/rest_j/v1/entrance/${execID}/log",
+ "status": 0,
+ "message": "Return progress information",
+ "data": {
+ "execID": "${execID}",
+ "taskID":28594,
+ "progress": 0.2,
+ "progressInfo": [
+ {
+ "id": "job-1",
+ "succeedTasks": 2,
+ "failedTasks": 0,
+ "runningTasks": 5,
+ "totalTasks": 10
+ },
+ {
+ "id": "job-2",
+ "succeedTasks": 5,
+ "failedTasks": 0,
+ "runningTasks": 5,
+ "totalTasks": 10
+ }
+ ]
+ },
+ "websocketTag":"37fcbd8b762d465a0c870684a0261c6e"
+}
+```
+
+
+
+##### 2.3HTTP API description
+For HTTP API, polling should be used to retrieve the status, logs and progress information after submission.
+
+
+**1).Request execution**
+
+- API `/api/rest_j/entrance/execute`
+- HTTP Method `POST`<br>
+- Sample JSON request body, its description is the same with Table 1
+```json
+{
+ "params": {},
+ "executeApplicationName":"spark",
+ "executionCode":"show tables",
+ "runType":"sql",
+ "source":{
+ "scriptPath": "/home/linkis/linkis.sql"
+ }
+}
+```
+- The response is consistent with websocket. Both execId and taskId will be obtained.
+
+**2).Retrieve status**<br>
+<br>
+- API `/api/rest_j/entrance/${execID}/status`
+- HTTP Method `GET`<br>
+- Sample response body
+```json
+{
+ "method": "/api/rest_j/v1/entrance/{execID}/status",
+ "status": 0,
+ "message": "Succeeded to retrieve status",
+ "data": {
+ "execID": "${execID}",
+ "status": "Running"
+ }
+}
+```
+
+**3).Retrieve logs**<br>
+<br>
+- API `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}`
+- HTTP Method `GET`
+- Parameter fromLine specifies from which line to start. Parameter size specifies the number of lines should be retrieved for this request.
+- Sample response body, the returned fromLine indicates the value of parameter fromLine for next request.
+```json
+{
+ "method": "/api/rest_j/v1/entrance/${execID}/log",
+ "status": 0,
+ "message": "Return logs information",
+ "data": {
+ "execID": "${execID}",
+ "log": ["errorLogs","warnLogs","infoLogs", "allLogs],
+ "fromLine": 56
+ }
+}
+```
+
+**4).Retrieve progress**<br>
+<br>
+- API `/api/rest_j/entrance/${execID}/progress`
+- HTTP Method `GET`<br>
+- Sample response body
+```json
+{
+ "method": "/api/rest_j/v1/entrance/{execID}/progress",
+ "status": 0,
+ "message": "Return progress information",
+ "data": {
+ "execID": "${execID}",
+ "progress": 0.2,
+ "progressInfo": [
+ {
+ "id": "job-1",
+ "succeedTasks": 2,
+ "failedTasks": 0,
+ "runningTasks": 5,
+ "totalTasks": 10
+ },
+ {
+ "id": "job-2",
+ "succeedTasks": 5,
+ "failedTasks": 0,
+ "runningTasks": 5,
+ "totalTasks": 10
+ }
+ ]
+ }
+}
+```
+**5).kill task**<br>
+<br>
+- API `/api/rest_j/entrance/${execID}/kill`
+- HTTP Method `POST`
+- Sample response body
+```json
+{
+ "method": "/api/rest_j/v1/entrance/{execID}/kill",
+ "status": 0,
+ "message": "OK",
+ "data": {
+ "execID":"${execID}"
+ }
+}
+```
+
+
+
+### 3.Client SDK Adaption
+[Please see Linkis Quick Start](/docs/en_US/ch2/Linkis%20Quick%20Start.md)
+
+
+
+#### 4. Multiple engine type support
+Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. The purpose and architecture of these three modules please refer to Linkis Architect Design Docs.
+
+#### Convention
+
+Linkis uses Spring framework as the underlying technique. So instances of some classes can be injected using Spring annotations. Linkis provides some default common implementations. If customized classes are needed by users, they can be directly injected and replace the current implementations.
+
+##### 4.1Entrance module adaption
+**1)maven dependency**
+```xml
+<dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-ujes-entrance</artifactId>
+ <version>0.9.4</version>
+</dependency>
+```
+**2)Interfaces to be implemented**
+
+There is no compulsory interface in Entrance. Below interfaces can be implemented on demand.
+- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overridden. Linkis provides CommonEntranceParser as the default implementation.
+- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager.
+- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes.
+
+##### 4.2EngineManager module adaption
+**1)maven dependency**
+```xml
+<dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-ujes-enginemanager</artifactId>
+ <version>0.9.4</version>
+</dependency>
+```
+
+**2)Interfaces to be implemented**
+
+Below interfaces are required to be implemented in EngineManager:
+- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder.
+Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified.
+- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements.
+- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to specify an Array[EngineHook] for dependency injection.
+- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection.
+
+##### 4.3Engine module adaption
+**1)maven dependency**
+```xml
+<dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-ujes-engine</artifactId>
+ <version>0.9.4</version>
+</dependency>
+```
+**2)Interfaces to be implemented**
+
+Below interfaces are required to be implemented in Engine：
+- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor.
+- EngineExecutor. The actual executor to execute the code submitted from the entrance. Methods need to be implemented: getActualUsedResources(the resource an engine acually used), executeLine(execute a line of the code parsed by CodeParser), executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine)
+
+Below interfaces/beans are optional in Engine:
+- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default.
+- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default.
- EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob.
\ No newline at end of file
diff --git a/docs/en_US/ch3/Linkis_Java_SDK_doc.md b/docs/en_US/ch3/Linkis_Java_SDK_doc.md
index 244e7a4..487d782 100644
--- a/docs/en_US/ch3/Linkis_Java_SDK_doc.md
+++ b/docs/en_US/ch3/Linkis_Java_SDK_doc.md
@@ -1,157 +1,157 @@
-### Linkis Java SDK Doc
-#### 1.Introduction
-
-Linkis provides a Java client implementation for users to have quick-access to Linkis background services using UJESClient.
-
-#### 2 Quick start
-
- We provided two test classes under dir ujes/client/src/test：
-
- com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTestJ # Test class based on Java
- com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTest # Test class based on Scala
-
- If you have cloned the source code of Linkis, you can directly run these two test classes.
-
-
- **Below sections introduce about how to write the code to complete a single execution on Linkis**
-
-#### 3 Fast implementation
-
-##### 3.1 maven dependency
-
-```xml
-<dependency>
- <groupId>com.webank.wedatasphere.Linkis</groupId>
- <artifactId>Linkis-ujes-client</artifactId>
- <version>0.9.3</version>
-</dependency>
-```
-
-#### 3.2 Sample implementation
-
-- **JAVA**
-```java
-package com.webank.bdp.dataworkcloud.ujes.client;
-
-import com.webank.wedatasphere.Linkis.common.utils.Utils;
-import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy;
-import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfig;
-import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder;
-import com.webank.wedatasphere.Linkis.ujes.client.UJESClient;
-import com.webank.wedatasphere.Linkis.ujes.client.UJESClientImpl;
-import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction;
-import com.webank.wedatasphere.Linkis.ujes.client.request.ResultSetAction;
-import com.webank.wedatasphere.Linkis.ujes.client.response.JobExecuteResult;
-import com.webank.wedatasphere.Linkis.ujes.client.response.JobInfoResult;
-import com.webank.wedatasphere.Linkis.ujes.client.response.JobProgressResult;
-import com.webank.wedatasphere.Linkis.ujes.client.response.JobStatusResult;
-import org.apache.commons.io.IOUtils;
-
-import java.util.concurrent.TimeUnit;
-
-
-public class UJESClientImplTestJ{
- public static void main(String[] args){
- // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder.
- DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder()
- .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port}
- .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client
- .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered.
- .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled.
- .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold
- .retryEnabled(false).readTimeout(30000) //whether to retry after failure
- .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis
- .setAuthTokenKey("${username}").setAuthTokenValue("${password}"))) //The authentication key，usually the username;The authentication value，usually the password
- .setDWSVersion("v1").build(); //The version of Linkis background protocol, currently v1
-
- // 2. Create a UJESClient from DWSClientConfig
- UJESClient client = new UJESClientImpl(clientConfig);
-
- // 3. Begin to execute the code
- JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder()
- .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation.
- .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed
- .setEngineType(JobExecuteAction.EngineType$.MODULE$.HIVE()) // The engine type expected by the client, i.e. Spark, Hive, etc...
- .setUser("johnnwang") //User, The user who makes this request；Used for user level multi-tenant isolation
- .build());
- System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID());
-
- // 4. Synch the status of script execution
- JobStatusResult status = client.status(jobExecuteResult);
- while(!status.isCompleted()) {
- // 5. Synch the status of script execution
- JobProgressResult progress = client.progress(jobExecuteResult);
- Utils.sleepQuietly(500);
- status = client.status(jobExecuteResult);
- }
-
- // 6. Synch the job information of script execution
- JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult);
- // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once)
- String resultSet = jobInfo.getResultSetList(client)[0];
- // 8. Fetch detailed result set content with a particular result set info
- Object fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()).getFileContent();
- System.out.println("fileContents: " + fileContents);
- IOUtils.closeQuietly(client);
- }
-}
-```
-
-- **SCALA**
-```scala
-
-import java.util.concurrent.TimeUnit
-
-import com.webank.wedatasphere.Linkis.common.utils.Utils
-import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy
-import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder
-import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction.EngineType
-import com.webank.wedatasphere.Linkis.ujes.client.request.{JobExecuteAction, ResultSetAction}
-import org.apache.commons.io.IOUtils
-
-object UJESClientImplTest extends App {
-
- // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder.
- val clientConfig = DWSClientConfigBuilder.newBuilder()
- .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port}
- .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client
- .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered.
- .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled.
- .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold
- .retryEnabled(false).readTimeout(30000) //whether to retry after failure
- .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis
- .setAuthTokenKey("${username}").setAuthTokenValue("${password}") //The authentication key，usually the username;The authentication value，usually the password
- .setDWSVersion("v1").build() //The version of Linkis background protocol, currently v1
-
- // 2. Create a UJESClient from DWSClientConfig
- val client = UJESClient(clientConfig)
-
- // 3. Begin to execute the code
- val jobExecuteResult = client.execute(JobExecuteAction.builder()
- .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation.
- .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed
- .setEngineType(EngineType.SPARK) // The engine type expected by the client, i.e. Spark, Hive, etc...
- .setUser("${username}").build()) //User, The user who makes this request；Used for user level multi-tenant isolation
- println("execId: " + jobExecuteResult.getExecID + ", taskId: " + jobExecuteResult.taskID)
-
- // 4. Synch the status of script execution
- var status = client.status(jobExecuteResult)
- while(!status.isCompleted) {
- // 5. Synch the status of script execution
- val progress = client.progress(jobExecuteResult)
- val progressInfo = if(progress.getProgressInfo != null) progress.getProgressInfo.toList else List.empty
- println("progress: " + progress.getProgress + ", progressInfo: " + progressInfo)
- Utils.sleepQuietly(500)
- status = client.status(jobExecuteResult)
- }
-
- // 6. Synch the job information of script execution
- val jobInfo = client.getJobInfo(jobExecuteResult)
- // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once)
- val resultSet = jobInfo.getResultSetList(client).head
- // 8. Fetch detailed result set content with a particular result set info
- val fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser).build()).getFileContent
- println("fileContents: " + fileContents)
- IOUtils.closeQuietly(client)
-}
-```
+### Linkis Java SDK Doc
+#### 1.Introduction
+
+Linkis provides a Java client implementation for users to have quick-access to Linkis background services using UJESClient.
+
+#### 2 Quick start
+
+ We provided two test classes under dir ujes/client/src/test：
+
+ com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTestJ # Test class based on Java
+ com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTest # Test class based on Scala
+
+ If you have cloned the source code of Linkis, you can directly run these two test classes.
+
+
+ **Below sections introduce about how to write the code to complete a single execution on Linkis**
+
+#### 3 Fast implementation
+
+##### 3.1 maven dependency
+
+```xml
+<dependency>
+ <groupId>com.webank.wedatasphere.Linkis</groupId>
+ <artifactId>Linkis-ujes-client</artifactId>
+ <version>0.9.4</version>
+</dependency>
+```
+
+#### 3.2 Sample implementation
+
+- **JAVA**
+```java
+package com.webank.bdp.dataworkcloud.ujes.client;
+
+import com.webank.wedatasphere.Linkis.common.utils.Utils;
+import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy;
+import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfig;
+import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder;
+import com.webank.wedatasphere.Linkis.ujes.client.UJESClient;
+import com.webank.wedatasphere.Linkis.ujes.client.UJESClientImpl;
+import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction;
+import com.webank.wedatasphere.Linkis.ujes.client.request.ResultSetAction;
+import com.webank.wedatasphere.Linkis.ujes.client.response.JobExecuteResult;
+import com.webank.wedatasphere.Linkis.ujes.client.response.JobInfoResult;
+import com.webank.wedatasphere.Linkis.ujes.client.response.JobProgressResult;
+import com.webank.wedatasphere.Linkis.ujes.client.response.JobStatusResult;
+import org.apache.commons.io.IOUtils;
+
+import java.util.concurrent.TimeUnit;
+
+
+public class UJESClientImplTestJ{
+ public static void main(String[] args){
+ // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder.
+ DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder()
+ .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port}
+ .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client
+ .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered.
+ .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled.
+ .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold
+ .retryEnabled(false).readTimeout(30000) //whether to retry after failure
+ .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis
+ .setAuthTokenKey("${username}").setAuthTokenValue("${password}"))) //The authentication key，usually the username;The authentication value，usually the password
+ .setDWSVersion("v1").build(); //The version of Linkis background protocol, currently v1
+
+ // 2. Create a UJESClient from DWSClientConfig
+ UJESClient client = new UJESClientImpl(clientConfig);
+
+ // 3. Begin to execute the code
+ JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder()
+ .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation.
+ .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed
+ .setEngineType(JobExecuteAction.EngineType$.MODULE$.HIVE()) // The engine type expected by the client, i.e. Spark, Hive, etc...
+ .setUser("johnnwang") //User, The user who makes this request；Used for user level multi-tenant isolation
+ .build());
+ System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID());
+
+ // 4. Synch the status of script execution
+ JobStatusResult status = client.status(jobExecuteResult);
+ while(!status.isCompleted()) {
+ // 5. Synch the status of script execution
+ JobProgressResult progress = client.progress(jobExecuteResult);
+ Utils.sleepQuietly(500);
+ status = client.status(jobExecuteResult);
+ }
+
+ // 6. Synch the job information of script execution
+ JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult);
+ // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once)
+ String resultSet = jobInfo.getResultSetList(client)[0];
+ // 8. Fetch detailed result set content with a particular result set info
+ Object fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()).getFileContent();
+ System.out.println("fileContents: " + fileContents);
+ IOUtils.closeQuietly(client);
+ }
+}
+```
+
+- **SCALA**
+```scala
+
+import java.util.concurrent.TimeUnit
+
+import com.webank.wedatasphere.Linkis.common.utils.Utils
+import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy
+import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder
+import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction.EngineType
+import com.webank.wedatasphere.Linkis.ujes.client.request.{JobExecuteAction, ResultSetAction}
+import org.apache.commons.io.IOUtils
+
+object UJESClientImplTest extends App {
+
+ // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder.
+ val clientConfig = DWSClientConfigBuilder.newBuilder()
+ .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port}
+ .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client
+ .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered.
+ .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled.
+ .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold
+ .retryEnabled(false).readTimeout(30000) //whether to retry after failure
+ .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis
+ .setAuthTokenKey("${username}").setAuthTokenValue("${password}") //The authentication key，usually the username;The authentication value，usually the password
+ .setDWSVersion("v1").build() //The version of Linkis background protocol, currently v1
+
+ // 2. Create a UJESClient from DWSClientConfig
+ val client = UJESClient(clientConfig)
+
+ // 3. Begin to execute the code
+ val jobExecuteResult = client.execute(JobExecuteAction.builder()
+ .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation.
+ .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed
+ .setEngineType(EngineType.SPARK) // The engine type expected by the client, i.e. Spark, Hive, etc...
+ .setUser("${username}").build()) //User, The user who makes this request；Used for user level multi-tenant isolation
+ println("execId: " + jobExecuteResult.getExecID + ", taskId: " + jobExecuteResult.taskID)
+
+ // 4. Synch the status of script execution
+ var status = client.status(jobExecuteResult)
+ while(!status.isCompleted) {
+ // 5. Synch the status of script execution
+ val progress = client.progress(jobExecuteResult)
+ val progressInfo = if(progress.getProgressInfo != null) progress.getProgressInfo.toList else List.empty
+ println("progress: " + progress.getProgress + ", progressInfo: " + progressInfo)
+ Utils.sleepQuietly(500)
+ status = client.status(jobExecuteResult)
+ }
+
+ // 6. Synch the job information of script execution
+ val jobInfo = client.getJobInfo(jobExecuteResult)
+ // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once)
+ val resultSet = jobInfo.getResultSetList(client).head
+ // 8. Fetch detailed result set content with a particular result set info
+ val fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser).build()).getFileContent
+ println("fileContents: " + fileContents)
+ IOUtils.closeQuietly(client)
+}
+```
diff --git "a/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md"
index 3553a9e..4172794 100644
--- "a/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md"
+++ "b/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md"
@@ -21,7 +21,7 @@
<dependency>
<groupId>com.webank.wedatasphere.Linkis</groupId>
<artifactId>Linkis-ujes-client</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</dependency>
```
diff --git "a/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" "b/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md"
index 6b3c972..42f3cdb 100644
--- "a/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md"
+++ "b/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md"
@@ -22,7 +22,7 @@
<dependency>
<groupId>com.webank.wedatasphere.Linkis</groupId>
<artifactId>Linkis-ujes-entrance</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</dependency>
```
@@ -42,7 +42,7 @@
<dependency>
<groupId>com.webank.wedatasphere.Linkis</groupId>
<artifactId>Linkis-ujes-enginemanager</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</dependency>
```
@@ -74,7 +74,7 @@
<dependency>
<groupId>com.webank.wedatasphere.Linkis</groupId>
<artifactId>Linkis-ujes-engine</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</dependency>
```
diff --git "a/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md"
index 0695c79..1c61591 100644
--- "a/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md"
+++ "b/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md"
@@ -312,7 +312,7 @@
<dependency>
<groupId>com.webank.wedatasphere.Linkis</groupId>
<artifactId>Linkis-ujes-entrance</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</dependency>
```
**2)需要实现的接口**
@@ -328,7 +328,7 @@
<dependency>
<groupId>com.webank.wedatasphere.Linkis</groupId>
<artifactId>Linkis-ujes-enginemanager</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</dependency>
```
@@ -348,7 +348,7 @@
<dependency>
<groupId>com.webank.wedatasphere.Linkis</groupId>
<artifactId>Linkis-ujes-engine</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</dependency>
```
**2)需要实现的接口**
diff --git a/eurekaServer/pom.xml b/eurekaServer/pom.xml
index 2f41280..2999d30 100644
--- a/eurekaServer/pom.xml
+++ b/eurekaServer/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-eureka-server</artifactId>
diff --git a/extensions/spark-excel/pom.xml b/extensions/spark-excel/pom.xml
index 10ce736..0c920b6 100644
--- a/extensions/spark-excel/pom.xml
+++ b/extensions/spark-excel/pom.xml
@@ -6,7 +6,7 @@
<groupId>com.webank.weDataSphere</groupId>
<artifactId>spark-excel_2.11</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/gateway/core/pom.xml b/gateway/core/pom.xml
index 26bacc8..2b83561 100644
--- a/gateway/core/pom.xml
+++ b/gateway/core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-gateway-core</artifactId>
diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala
index 67eab5f..f8ef9a7 100644
--- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala
+++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.gateway.config
-import com.webank.wedatasphere.linkis.gateway.security.{LDAPUserRestful, SecurityHook, SecurityFilter, UserRestful}
+import com.webank.wedatasphere.linkis.gateway.security.{LDAPUserRestful, SecurityFilter, SecurityHook, UserRestful}
import javax.annotation.PostConstruct
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala
index 87906ee..092ece9 100644
--- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala
+++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala
@@ -16,6 +16,10 @@
package com.webank.wedatasphere.linkis.gateway.http
+import java.util
+
+import com.webank.wedatasphere.linkis.server.JMap
+
/**
* created by cooperyang on 2019/1/9.
*/
@@ -32,6 +36,7 @@
def setGatewayRoute(gatewayRoute: GatewayRoute): Unit
def getGatewayRoute: GatewayRoute
+ def getParams: JMap[String, String]
}
class BaseGatewayContext extends GatewayContext {
private var request: GatewayHttpRequest = _
@@ -39,6 +44,8 @@
private var webSocketRequest: Boolean = false
private var gatewayRoute: GatewayRoute = _
+ private val props: JMap[String, String] = new util.HashMap[String, String]()
+
override def getRequest: GatewayHttpRequest = request
override def setRequest(request: GatewayHttpRequest): Unit = this.request = request
@@ -54,4 +61,6 @@
override def setGatewayRoute(gatewayRoute: GatewayRoute): Unit = this.gatewayRoute = gatewayRoute
override def getGatewayRoute: GatewayRoute = gatewayRoute
+
+ override def getParams: JMap[String, String] = this.props
}
\ No newline at end of file
diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala
index 2d459d5..08dd795 100644
--- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala
+++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala
@@ -19,8 +19,8 @@
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.gateway.http.{GatewayContext, GatewayHttpRequest}
import com.webank.wedatasphere.linkis.server.exception.LoginExpireException
-import com.webank.wedatasphere.linkis.server.security.{SSOUtils, ServerSSOUtils}
import com.webank.wedatasphere.linkis.server.security.SecurityFilter._
+import com.webank.wedatasphere.linkis.server.security.{SSOUtils, ServerSSOUtils}
import javax.servlet.http.Cookie
import scala.collection.JavaConversions._
diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala
index 8678f53..d01b9db 100644
--- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala
+++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala
@@ -16,9 +16,7 @@
package com.webank.wedatasphere.linkis.gateway.security
-import java.util.Random
-
-import com.google.gson.{Gson, JsonObject}
+import com.google.gson.Gson
import com.webank.wedatasphere.linkis.common.utils.{Logging, RSAUtils, Utils}
import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration
import com.webank.wedatasphere.linkis.gateway.http.GatewayContext
diff --git a/gateway/gateway-httpclient-support/pom.xml b/gateway/gateway-httpclient-support/pom.xml
index ca6a715..c0daf49 100644
--- a/gateway/gateway-httpclient-support/pom.xml
+++ b/gateway/gateway-httpclient-support/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-gateway-httpclient-support</artifactId>
@@ -42,7 +42,7 @@
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
- <version>1.7.0</version>
+ <version>1.9.4</version>
</dependency>
<dependency>
<groupId>org.reflections</groupId>
diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala
index 6f9c481..d36005c 100644
--- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala
+++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala
@@ -24,7 +24,6 @@
import java.util
import com.fasterxml.jackson.databind.ObjectMapper
-import com.ning.http.client.Response
import com.webank.wedatasphere.linkis.common.io.{Fs, FsPath}
import com.webank.wedatasphere.linkis.httpclient.AbstractHttpClient
import com.webank.wedatasphere.linkis.httpclient.discovery.Discovery
@@ -37,6 +36,7 @@
import com.webank.wedatasphere.linkis.storage.FSFactory
import org.apache.commons.beanutils.BeanUtils
import org.apache.commons.lang.ClassUtils
+import org.apache.http.HttpResponse
import scala.collection.JavaConversions.mapAsJavaMap
@@ -44,7 +44,7 @@
* created by cooperyang on 2019/5/20.
*/
class DWSHttpClient(clientConfig: DWSClientConfig, clientName: String)
- extends AbstractHttpClient(clientConfig, clientName) {
+ extends AbstractHttpClient(clientConfig, clientName) {
override protected def createDiscovery(): Discovery = new DWSGatewayDiscovery
@@ -57,21 +57,25 @@
requestAction
}
- override protected def httpResponseToResult(response: Response, requestAction: HttpAction): Option[Result] = {
- val url = requestAction.getURL
+ override protected def httpResponseToResult(response: HttpResponse, requestAction: HttpAction, responseBody: String): Option[Result] = {
+ var entity = response.getEntity
+ val statusCode: Int = response.getStatusLine.getStatusCode
+ val url: String = requestAction.getURL
+ val contentType: String = entity.getContentType.getValue
DWSHttpMessageFactory.getDWSHttpMessageResult(url).map { case DWSHttpMessageResultInfo(_, clazz) =>
clazz match {
case c if ClassUtils.isAssignable(c, classOf[DWSResult]) =>
val dwsResult = clazz.getConstructor().newInstance().asInstanceOf[DWSResult]
- dwsResult.set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType)
+ dwsResult.set(responseBody, statusCode, url, contentType)
BeanUtils.populate(dwsResult, dwsResult.getData)
return Some(dwsResult)
case _ =>
}
+
def transfer(value: Result, map: Map[String, Object]): Unit = {
value match {
case httpResult: HttpResult =>
- httpResult.set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType)
+ httpResult.set(responseBody, statusCode, url, contentType)
case _ =>
}
val javaMap = mapAsJavaMap(map)
@@ -89,17 +93,18 @@
transfer(value, map)
value
}.toArray
- new ListResult(response.getResponseBody, results)
+ new ListResult(responseBody, results)
}
}.orElse(nonDWSResponseToResult(response, requestAction))
}
- protected def nonDWSResponseToResult(response: Response, requestAction: HttpAction): Option[Result] = None
+ protected def nonDWSResponseToResult(response: HttpResponse, requestAction: HttpAction): Option[Result] = None
protected def fillResultFields(responseMap: util.Map[String, Object], value: Result): Unit = {}
//TODO Consistent with workspace, plus expiration time(与workspace保持一致，加上过期时间)
override protected def getFsByUser(user: String, path: FsPath): Fs = FSFactory.getFsByProxyUser(path, user)
+
}
object DWSHttpClient {
val jacksonJson = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"))
diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala
index ab967b9..129a2a0 100644
--- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala
+++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.httpclient.dws.authentication
-import com.ning.http.client.Response
import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils
import com.webank.wedatasphere.linkis.httpclient.authentication.{AbstractAuthenticationStrategy, AuthenticationAction, AuthenticationResult}
import com.webank.wedatasphere.linkis.httpclient.dws.exception.AuthenticationFailedException
@@ -24,6 +23,7 @@
import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSAuthenticationResult
import com.webank.wedatasphere.linkis.httpclient.request.{Action, UserAction, UserPwdAction}
import org.apache.commons.lang.StringUtils
+import org.apache.http.HttpResponse
/**
* created by cooperyang on 2019/5/22.
@@ -51,7 +51,7 @@
action
}
- override def getAuthenticationResult(response: Response, requestAction: AuthenticationAction): AuthenticationResult = {
+ override def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult = {
new DWSAuthenticationResult(response, requestAction.serverUrl)
}
}
\ No newline at end of file
diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala
index ac74199..061bcc2 100644
--- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala
+++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala
@@ -18,11 +18,11 @@
import java.util
-import com.ning.http.client.Response
-import com.ning.http.client.cookie.Cookie
import com.webank.wedatasphere.linkis.httpclient.authentication._
import com.webank.wedatasphere.linkis.httpclient.dws.exception.AuthenticationFailedException
import com.webank.wedatasphere.linkis.httpclient.request.{Action, UserAction}
+import org.apache.http.HttpResponse
+import org.apache.http.cookie.Cookie
/**
* Created by enjoyyin on 2019/10/4.
@@ -35,6 +35,7 @@
case _: AuthenticationAction => null
case action: UserAction => new HttpAuthentication {
import TokenAuthenticationStrategy._
+
import scala.collection.JavaConversions._
override def authToCookies: Array[Cookie] = Array.empty
@@ -51,7 +52,7 @@
override protected def getAuthenticationAction(requestAction: Action, serverUrl: String): AuthenticationAction = null
- override def getAuthenticationResult(response: Response, requestAction: AuthenticationAction): AuthenticationResult = null
+ override def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult = null
}
object TokenAuthenticationStrategy {
diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala
index 51481c7..2498b8e 100644
--- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala
+++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.httpclient.dws.config
-import com.webank.wedatasphere.linkis.httpclient.config.{ClientConfig, ClientConfigBuilder}
+import com.webank.wedatasphere.linkis.httpclient.config.ClientConfigBuilder
import com.webank.wedatasphere.linkis.httpclient.dws.exception.UnknownVersionException
import org.apache.commons.lang.StringUtils
diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala
index 2dc2992..bb77b5b 100644
--- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala
+++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala
@@ -20,10 +20,10 @@
package com.webank.wedatasphere.linkis.httpclient.dws.discovery
-import com.ning.http.client.Response
import com.webank.wedatasphere.linkis.httpclient.discovery.{AbstractDiscovery, HeartbeatAction, HeartbeatResult}
import com.webank.wedatasphere.linkis.httpclient.dws.request.DWSHeartbeatAction
import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSHeartbeatResult
+import org.apache.http.HttpResponse
import scala.util.Random
@@ -43,7 +43,7 @@
override protected def getHeartbeatAction(serverUrl: String): HeartbeatAction = new DWSHeartbeatAction(serverUrl)
- override def getHeartbeatResult(response: Response, requestAction: HeartbeatAction): HeartbeatResult = requestAction match {
+ override def getHeartbeatResult(response: HttpResponse, requestAction: HeartbeatAction): HeartbeatResult = requestAction match {
case h: DWSHeartbeatAction => new DWSHeartbeatResult(response, h.serverUrl)
}
}
diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala
index de939f8..bc2c479 100644
--- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala
+++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala
@@ -18,22 +18,37 @@
import java.util
-import com.ning.http.client.Response
-import com.ning.http.client.cookie.Cookie
import com.webank.wedatasphere.linkis.httpclient.authentication.{Authentication, AuthenticationResult, HttpAuthentication}
import com.webank.wedatasphere.linkis.httpclient.exception.HttpMessageParseException
-
-import scala.collection.JavaConversions
+import org.apache.http.HttpResponse
+import org.apache.http.cookie.Cookie
+import org.apache.http.util.EntityUtils
/**
* created by cooperyang on 2019/5/22.
*/
-class DWSAuthenticationResult(response: Response, serverUrl: String) extends AuthenticationResult with DWSResult {
+class DWSAuthenticationResult(response: HttpResponse, serverUrl: String) extends AuthenticationResult with DWSResult {
- set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType)
- override def getAuthentication: Authentication = if(getStatus == 0) new HttpAuthentication {
+ setResponse()
+
+ private def setResponse(): Unit = {
+ val entity = response.getEntity
+ val responseBody: String = if (entity != null) {
+ EntityUtils.toString(entity, "UTF-8")
+ } else {
+ null
+ }
+ val statusCode: Int = response.getStatusLine.getStatusCode
+ val url: String = serverUrl
+ val contentType: String = entity.getContentType.getValue
+ set(responseBody, statusCode, url, contentType)
+ }
+
+
+ override def getAuthentication: Authentication = if (getStatus == 0) new HttpAuthentication {
private var lastAccessTime: Long = System.currentTimeMillis
- override def authToCookies: Array[Cookie] = JavaConversions.asScalaBuffer(response.getCookies).toArray
+
+ override def authToCookies: Array[Cookie] = Array.empty
override def authToHeaders: util.Map[String, String] = new util.HashMap[String, String]()
@@ -44,4 +59,5 @@
override def updateLastAccessTime(): Unit = lastAccessTime = System.currentTimeMillis
} else throw new HttpMessageParseException(s"login to gateway $serverUrl failed! Reason: " + getMessage)
+
}
diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala
index 3697e33..4030a5a 100644
--- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala
+++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala
@@ -18,16 +18,33 @@
import java.util
-import com.ning.http.client.Response
import com.webank.wedatasphere.linkis.httpclient.discovery.HeartbeatResult
+import org.apache.http.HttpResponse
+import org.apache.http.util.EntityUtils
/**
* created by cooperyang on 2019/5/22.
*/
-class DWSHeartbeatResult(response: Response, serverUrl: String) extends HeartbeatResult with DWSResult {
+class DWSHeartbeatResult(response: HttpResponse, serverUrl: String) extends HeartbeatResult with DWSResult {
- set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType)
- if(getStatus != 0) warn(s"heartbeat to gateway $serverUrl failed! message: $getMessage.")
+
+ setResponse()
+
+ private def setResponse(): Unit = {
+ val entity = response.getEntity
+ val responseBody: String = if (entity != null) {
+ EntityUtils.toString(entity, "UTF-8")
+ } else {
+ null
+ }
+ val statusCode: Int = response.getStatusLine.getStatusCode
+ val url: String = serverUrl
+ val contentType: String = entity.getContentType.getValue
+ set(responseBody, statusCode, url, contentType)
+ }
+
+
+ if (getStatus != 0) warn(s"heartbeat to gateway $serverUrl failed! message: $getMessage.")
override val isHealthy: Boolean = getData.get("isHealthy") match {
case b: java.lang.Boolean => b
case s if s != null => s.toString.toBoolean
diff --git a/gateway/gateway-ujes-support/pom.xml b/gateway/gateway-ujes-support/pom.xml
index 9613d47..bc04195 100644
--- a/gateway/gateway-ujes-support/pom.xml
+++ b/gateway/gateway-ujes-support/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-gateway-ujes-support</artifactId>
@@ -51,6 +51,12 @@
<version>2.1</version>
</dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-common</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
</dependencies>
<build>
diff --git a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala
index 348934e..4775203 100644
--- a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala
+++ b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala
@@ -19,12 +19,11 @@
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.gateway.http.GatewayContext
import com.webank.wedatasphere.linkis.gateway.parser.AbstractGatewayParser
+import com.webank.wedatasphere.linkis.gateway.ujes.parser.EntranceExecutionGatewayParser._
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
import com.webank.wedatasphere.linkis.protocol.utils.ZuulEntranceUtils
import org.springframework.stereotype.Component
-import EntranceExecutionGatewayParser._
-
/**
* created by cooperyang on 2019/5/15.
*/
diff --git a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/HaContextGatewayRouter.scala b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/HaContextGatewayRouter.scala
new file mode 100644
index 0000000..39ddbe9
--- /dev/null
+++ b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/HaContextGatewayRouter.scala
@@ -0,0 +1,124 @@
+package com.webank.wedatasphere.linkis.gateway.ujes.route
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.cs.common.entity.source.{ContextID, ContextIDParser}
+import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant
+import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper
+import com.webank.wedatasphere.linkis.gateway.http.GatewayContext
+import com.webank.wedatasphere.linkis.gateway.route.AbstractGatewayRouter
+import com.webank.wedatasphere.linkis.gateway.springcloud.SpringCloudGatewayConfiguration.{API_URL_PREFIX, normalPath}
+import com.webank.wedatasphere.linkis.rpc.interceptor.ServiceInstanceUtils
+import org.apache.commons.lang.StringUtils
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Component
+
+import scala.collection.JavaConversions._
+import scala.util.Random
+import scala.util.matching.Regex
+
+/**
+ * created by cooperyang on 2020/2/19
+ * Description:
+ */
+@Component
+class HaContextGatewayRouter extends AbstractGatewayRouter{
+
+ @Autowired
+ private var contextIDParser: ContextIDParser = _
+ private val serializationHelper = ContextSerializationHelper.getInstance();
+
+ override def route(gatewayContext: GatewayContext): ServiceInstance = {
+
+ if (gatewayContext.getGatewayRoute.getRequestURI.contains("contextservice")){
+ val params: util.HashMap[String, String] = gatewayContext.getGatewayRoute.getParams
+ if (!gatewayContext.getRequest.getQueryParams.isEmpty) {
+ for ((k, vArr) <- gatewayContext.getRequest.getQueryParams) {
+ if (vArr.nonEmpty) {
+ params.putIfAbsent(k, vArr.head)
+ }
+ }
+ }
+ if (gatewayContext.getRequest.getHeaders.containsKey(ContextHTTPConstant.CONTEXT_ID_STR)) {
+ params.putIfAbsent(ContextHTTPConstant.CONTEXT_ID_STR, gatewayContext.getRequest.getHeaders.get(ContextHTTPConstant.CONTEXT_ID_STR)(0))
+ }
+ if (null == params || params.isEmpty) {
+ dealContextCreate(gatewayContext)
+ } else {
+ var contextId : String = null
+ for ((key, value) <- params) {
+ if (key.equalsIgnoreCase(ContextHTTPConstant.CONTEXT_ID_STR)) {
+ contextId = value
+ }
+ }
+ if (StringUtils.isNotBlank(contextId)) {
+ dealContextAccess(contextId.toString, gatewayContext)
+ } else {
+ dealContextCreate(gatewayContext)
+ }
+ }
+ }else{
+ null
+ }
+ }
+
+ def dealContextCreate(gatewayContext:GatewayContext):ServiceInstance = {
+ val serviceId = findService(HaContextGatewayRouter.CONTEXT_SERVICE_STR, list => {
+ val services = list.filter(_.contains(HaContextGatewayRouter.CONTEXT_SERVICE_STR))
+ services.headOption
+ })
+ val serviceInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances(serviceId.orNull)
+ if (serviceInstances.size > 0) {
+ val index = new Random().nextInt(serviceInstances.size)
+ serviceInstances(index)
+ } else {
+ logger.error(s"No valid instance for service : " + serviceId.orNull)
+ null
+ }
+ }
+
+ def dealContextAccess(contextIdStr:String, gatewayContext: GatewayContext):ServiceInstance = {
+ val contextId : String = {
+ var tmpId : String = null
+ if (serializationHelper.accepts(contextIdStr)) {
+ val contextID : ContextID = serializationHelper.deserialize(contextIdStr).asInstanceOf[ContextID]
+ if (null != contextID) {
+ tmpId = contextID.getContextId
+ } else {
+ error(s"Deserializate contextID null. contextIDStr : " + contextIdStr)
+ }
+ } else {
+ error(s"ContxtIDStr cannot be deserialized. contextIDStr : " + contextIdStr)
+ }
+ if (null == tmpId) {
+ contextIdStr
+ } else {
+ tmpId
+ }
+ }
+ val instances = contextIDParser.parse(contextId)
+ var serviceId:Option[String] = None
+ serviceId = findService(HaContextGatewayRouter.CONTEXT_SERVICE_STR, list => {
+ val services = list.filter(_.contains(HaContextGatewayRouter.CONTEXT_SERVICE_STR))
+ services.headOption
+ })
+ val serviceInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances(serviceId.orNull)
+ if (instances.size() > 0) {
+ serviceId.map(ServiceInstance(_, instances.get(0))).orNull
+ } else if (serviceInstances.size > 0) {
+ serviceInstances(0)
+ } else {
+ logger.error(s"No valid instance for service : " + serviceId.orNull)
+ null
+ }
+ }
+
+}
+
+
+object HaContextGatewayRouter{
+ val CONTEXT_ID_STR:String = "contextId"
+ val CONTEXT_SERVICE_STR:String = "contextservice"
+ val CONTEXT_REGEX: Regex = (normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/contextservice/" + ".+").r
+}
diff --git a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/contextservice/ContextIdParserImpl.scala b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/contextservice/ContextIdParserImpl.scala
new file mode 100644
index 0000000..a0fed9e
--- /dev/null
+++ b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/contextservice/ContextIdParserImpl.scala
@@ -0,0 +1,52 @@
+package com.webank.wedatasphere.linkis.gateway.ujes.route.contextservice
+
+import java.util
+import java.util.{ArrayList, List}
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.common.entity.source.{ContextIDParser, HAContextID}
+import com.webank.wedatasphere.linkis.cs.common.utils.CSHighAvailableUtils
+import com.webank.wedatasphere.linkis.rpc.instancealias.InstanceAliasManager
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Component
+
+
+/**
+ * created by cooperyang on 2020/2/19
+ * Description: 如果id为HAID，则解析出对应的instance
+ */
+@Component
+class ContextIdParserImpl extends ContextIDParser with Logging {
+
+ @Autowired
+ var instanceAliasManager : InstanceAliasManager = _
+
+ override def parse(contextId: String): util.List[String] = {
+
+ if (CSHighAvailableUtils.checkHAIDBasicFormat(contextId)) {
+ val instances = new util.ArrayList[String](2)
+ val haContextID = CSHighAvailableUtils.decodeHAID(contextId)
+ if (instanceAliasManager.isInstanceAliasValid(haContextID.getInstance)) {
+ instances.add(instanceAliasManager.getInstanceByAlias(haContextID.getInstance).getInstance)
+ } else {
+ error(s"parse HAID instance invalid. haIDKey : " + contextId)
+ }
+ if (instanceAliasManager.isInstanceAliasValid(haContextID.getBackupInstance)) {
+ instances.add(instanceAliasManager.getInstanceByAlias(haContextID.getBackupInstance).getInstance)
+ } else {
+ error("parse HAID backupInstance invalid. haIDKey : " + contextId)
+ }
+ instances
+ } else {
+ new util.ArrayList[String](0)
+ }
+ }
+
+ private def isNumberic(s:String):Boolean = {
+ s.toCharArray foreach {
+ c => if (c < 48 || c >57) return false
+ }
+ true
+ }
+
+}
diff --git a/gateway/springcloudgateway/pom.xml b/gateway/springcloudgateway/pom.xml
index 15de471..068c01c 100644
--- a/gateway/springcloudgateway/pom.xml
+++ b/gateway/springcloudgateway/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-gateway-springcloudgateway</artifactId>
diff --git a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala
index 06b77a6..88b4cdf 100644
--- a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala
+++ b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala
@@ -18,7 +18,6 @@
import com.netflix.loadbalancer.Server
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.gateway.config.GatewaySpringConfiguration
import com.webank.wedatasphere.linkis.gateway.parser.{DefaultGatewayParser, GatewayParser}
diff --git a/metadata/bin/start-metadata.sh b/metadata/bin/start-metadata.sh
index 56cab6d..80cc775 100644
--- a/metadata/bin/start-metadata.sh
+++ b/metadata/bin/start-metadata.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/metadata/pom.xml b/metadata/pom.xml
index 42dcc78..d4a05f4 100644
--- a/metadata/pom.xml
+++ b/metadata/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java
index 16e05a9..99651ac 100644
--- a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java
+++ b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTablePartitionStatisticInfoVO.java
@@ -16,6 +16,7 @@
package com.webank.wedatasphere.linkis.metadata.domain.mdq.vo;
import java.util.ArrayList;
+import java.util.Date;
import java.util.List;
@@ -23,6 +24,15 @@
private Integer fileNum;
private String partitionSize;
private String name;
+ private Date modificationTime;
+
+ public Date getModificationTime() {
+ return modificationTime;
+ }
+
+ public void setModificationTime(Date modificationTime) {
+ this.modificationTime = modificationTime;
+ }
public String getName() {
return name;
diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java
index 09565cf..13192b9 100644
--- a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java
+++ b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/domain/mdq/vo/MdqTableStatisticInfoVO.java
@@ -15,7 +15,6 @@
*/
package com.webank.wedatasphere.linkis.metadata.domain.mdq.vo;
-import java.util.ArrayList;
import java.util.Date;
import java.util.List;
diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java
index 0634dfd..2986b57 100644
--- a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java
+++ b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/service/impl/MdqServiceImpl.java
@@ -17,7 +17,6 @@
import com.google.common.collect.Maps;
import com.google.gson.Gson;
-
import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils;
import com.webank.wedatasphere.linkis.hadoop.common.utils.HDFSUtils;
import com.webank.wedatasphere.linkis.metadata.dao.MdqDao;
@@ -84,6 +83,13 @@
mdqDao.insertTable(table);
List<MdqTableFieldsInfoBO> tableFieldsInfo = mdqTableBO.getTableFieldsInfo();
List<MdqField> mdqFieldList = DomainCoversionUtils.mdqTableFieldsInfoBOListToMdqFieldList(tableFieldsInfo, table.getId());
+ if(table.getPartitionTable() && table.getImport()){
+ //创建表是导入,并且是分区表的话,自动去掉最后一个ds列
+ List<MdqField> collect = mdqFieldList.stream().filter(f -> "ds".equals(f.getName())).collect(Collectors.toList());
+ if(collect.size() > 1){
+ mdqFieldList.remove(collect.get(1));
+ }
+ }
mdqDao.insertFields(mdqFieldList);
if (mdqTableBO.getImportInfo() != null) {
MdqTableImportInfoBO importInfo = mdqTableBO.getImportInfo();
@@ -237,6 +243,7 @@
mdqTablePartitionStatisticInfoVO.setName(new Path(path).getName());
mdqTablePartitionStatisticInfoVO.setFileNum(getTableFileNum(path));
mdqTablePartitionStatisticInfoVO.setPartitionSize(getTableSize(path));
+ mdqTablePartitionStatisticInfoVO.setModificationTime(getTableModificationTime(path));
FileStatus tableFile = getRootHdfs().getFileStatus(new Path(path));
FileStatus[] fileStatuses = getRootHdfs().listStatus(tableFile.getPath());
List<FileStatus> collect = Arrays.stream(fileStatuses).filter(f -> f.isDirectory()).collect(Collectors.toList());
@@ -246,6 +253,14 @@
return mdqTablePartitionStatisticInfoVO;
}
+ private Date getTableModificationTime(String tableLocation) throws IOException {
+ if (StringUtils.isNotBlank(tableLocation)) {
+ FileStatus tableFile = getRootHdfs().getFileStatus(new Path(tableLocation));
+ return new Date(tableFile.getModificationTime());
+ }
+ return null;
+ }
+
private int getPartitionsNum(String tableLocation) throws IOException {
int partitionsNum = 0;
if (StringUtils.isNotBlank(tableLocation)) {
diff --git a/pom.xml b/pom.xml
index 35d08eb..06a0f64 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<packaging>pom</packaging>
<name>Linkis Project Parent POM</name>
@@ -57,42 +57,41 @@
<modules>
- <module>core/common</module>
+ <module>core/common</module>
<module>core/cloudProtocol</module>
<module>core/scheduler</module>
<module>core/cloudModule</module>
<module>core/cloudRPC</module>
<module>core/cloudMybatis</module>
- <module>core/httpclient</module>
<module>core/hadoop-common</module>
+ <module>core/httpclient</module>
<module>storage/storage</module>
-
<module>bml/bmlcommon</module>
<module>bml/bmlclient</module>
+ <module>bml/bmlserver</module>
<module>resourceManager/resourcemanagercommon</module>
<module>resourceManager/resourcemanagerclient</module>
- <module>gateway/core</module>
- <module>gateway/springcloudgateway</module>
- <module>gateway/gateway-httpclient-support</module>
-
+ <module>resourceManager/resourcemanagerserver</module>
<module>publicService/udf</module>
<module>publicService/application</module>
<module>publicService/jobhistory</module>
<module>publicService/configuration</module>
<module>publicService/variable</module>
+ <module>contextservice/cs-common</module>
+ <module>contextservice/cs-client</module>
+ <module>contextservice/cs-listener</module>
+ <module>contextservice/cs-ujes-client</module>
+ <module>contextservice/cs-cache</module>
+ <module>contextservice/cs-persistence</module>
+ <module>contextservice/cs-highavailable</module>
+ <module>ujes/entrance</module>
+ <module>ujes/entranceclient</module>
<module>publicService/workspace</module>
<module>publicService/workspace/client/workspace-httpclient</module>
- <module>bml/bmlserver</module>
- <module>resourceManager/resourcemanagerserver</module>
<module>metadata</module>
- <module>gateway/gateway-ujes-support</module>
- <module>eurekaServer</module>
- <module>publicService</module>
- <module>userControl</module>
<module>ujes/engine</module>
- <module>ujes/enginemanager</module>
- <module>ujes/entrance</module>
<module>bml/bml-engine-hook</module>
+ <module>ujes/enginemanager</module>
<module>ujes/definedEngines/spark/engine</module>
<module>ujes/definedEngines/spark/enginemanager</module>
<module>ujes/definedEngines/spark/entrance</module>
@@ -104,15 +103,34 @@
<module>ujes/definedEngines/python/engine</module>
<module>ujes/definedEngines/python/enginemanager</module>
<module>ujes/definedEngines/python/entrance</module>
- <module>ujes/definedEngines/shell/engine</module>
- <module>ujes/definedEngines/shell/enginemanager</module>
- <module>ujes/definedEngines/shell/entrance</module>
- <module>ujes/definedEngines/jdbc/entrance</module>
- <module>ujes/client</module>
+
<module>ujes/definedEngines/pipeline/engine</module>
<module>ujes/definedEngines/pipeline/enginemanager</module>
<module>ujes/definedEngines/pipeline/entrance</module>
+ <module>ujes/definedEngines/jdbc/entrance</module>
<module>ujes/definedEngines/mlsql/entrance</module>
+
+ <module>ujes/definedEngines/shell/engine</module>
+ <module>ujes/definedEngines/shell/entrance</module>
+ <module>ujes/definedEngines/shell/enginemanager</module>
+
+
+ <module>gateway/core</module>
+ <module>gateway/springcloudgateway</module>
+ <module>gateway/gateway-ujes-support</module>
+ <module>gateway/gateway-httpclient-support</module>
+ <module>eurekaServer</module>
+ <module>publicService</module>
+ <module>ujes/client</module>
+ <module>contextservice/cs-server</module>
+ <module>contextservice/cs-search</module>
+ <module>datasource/metadatamanager/common</module>
+ <module>datasource/metadatamanager/server</module>
+ <module>datasource/metadatamanager/service/mysql</module>
+ <module>datasource/metadatamanager/service/elasticsearch</module>
+ <module>datasource/metadatamanager/service/hive</module>
+ <module>datasource/datasourcemanager/common</module>
+ <module>datasource/datasourcemanager/server</module>
<module>assembly/public-module</module>
<module>assembly</module>
@@ -124,7 +142,7 @@
<spring.eureka.version>1.4.4.RELEASE</spring.eureka.version>
<spring.boot.version>2.0.3.RELEASE</spring.boot.version>
<gson.version>2.8.5</gson.version>
- <fasterxml.jackson.version>2.9.6</fasterxml.jackson.version>
+ <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
<scala.version>2.11.8</scala.version>
<jdk.compile.version>1.8</jdk.compile.version>
<plugin.scala.version>2.15.2</plugin.scala.version>
@@ -133,12 +151,12 @@
<jersey.servlet.version>2.23.1</jersey.servlet.version>
<jetty.version>9.4.11.v20180605</jetty.version>
<httpclient.version>4.5.4</httpclient.version>
+ <httpmime.version>4.5.4</httpmime.version>
<slf4j.version>1.7.12</slf4j.version>
- <linkis.version>0.9.3</linkis.version>
+ <linkis.version>0.9.4</linkis.version>
<maven.version>3.3.3</maven.version>
+ <xstream.core.version>1.4.11.1</xstream.core.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <dispatch.version>0.11.2</dispatch.version>
- <jsoup.version>1.8.2</jsoup.version>
</properties>
<dependencyManagement>
@@ -184,6 +202,11 @@
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.thoughtworks.xstream</groupId>
+ <artifactId>xstream</artifactId>
+ <version>${xstream.core.version}</version>
+ </dependency>
<!-- scala version -->
<dependency>
@@ -214,7 +237,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0</version>
+ <version>25.1-jre</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -238,7 +261,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <project.release.version>0.9.3</project.release.version>
+ <project.release.version>0.9.4</project.release.version>
<netty.version>4.1.17.Final</netty.version>
<json4s.version>3.5.3</json4s.version>
</properties>
@@ -246,7 +269,7 @@
<profile>
<id>spark2.3</id>
<properties>
- <project.release.version>0.9.3</project.release.version>
+ <project.release.version>0.9.4</project.release.version>
<netty.version>4.1.17.Final</netty.version>
<json4s.version>3.2.11</json4s.version>
</properties>
@@ -254,7 +277,7 @@
<profile>
<id>spark2.2-2.0</id>
<properties>
- <project.release.version>0.9.3</project.release.version>
+ <project.release.version>0.9.4</project.release.version>
<netty.version>4.0.47.Final</netty.version>
<json4s.version>3.2.11</json4s.version>
</properties>
@@ -264,44 +287,44 @@
<id>release</id>
<build>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- <version>3.1.0</version>
- <configuration>
- <attach>true</attach>
- </configuration>
- <executions>
- <execution>
- <id>create-source-jar</id>
- <goals>
- <goal>jar-no-fork</goal>
- <goal>test-jar-no-fork</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>3.1.0</version>
+ <configuration>
+ <attach>true</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <id>create-source-jar</id>
+ <goals>
+ <goal>jar-no-fork</goal>
+ <goal>test-jar-no-fork</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- <version>3.0.0-M1</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.1</version>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>3.0.0-M1</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>3.0.1</version>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
- </plugins>
+ </plugins>
</build>
</profile>
diff --git a/publicService/application/pom.xml b/publicService/application/pom.xml
index 01af0fe..7e921f7 100644
--- a/publicService/application/pom.xml
+++ b/publicService/application/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-application</artifactId>
@@ -47,7 +47,7 @@
<dependency>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis-storage</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<scope>provided</scope>
</dependency>
<dependency>
diff --git a/publicService/bin/start-publicservice.sh b/publicService/bin/start-publicservice.sh
index 56cab6d..80cc775 100644
--- a/publicService/bin/start-publicservice.sh
+++ b/publicService/bin/start-publicservice.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/publicService/configuration/pom.xml b/publicService/configuration/pom.xml
index 3150bd4..7fae3ee 100644
--- a/publicService/configuration/pom.xml
+++ b/publicService/configuration/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-configuration</artifactId>
diff --git a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/ConfigMapper.java b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/ConfigMapper.java
index ba59a45..0e51693 100644
--- a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/ConfigMapper.java
+++ b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/ConfigMapper.java
@@ -35,16 +35,28 @@
ConfigTree selectTreeByTreeID(@Param("id") Long id);
- List<ConfigKey> selectKeysByTreeID(@Param("treeID") Long treeID,@Param("creatorID") Long creatorID);
+ List<ConfigKey> selectKeysByTreeID(@Param("treeID") Long treeID, @Param("creatorID") Long creatorID);
void insertValue(ConfigKeyUser configKeyUser);
ConfigKeyUser selectValueByKeyId(@Param("keyID") Long keyID, @Param("userName") String userName, @Param("appID") Long appID);
- List<ConfigKeyValue> selectAllAppKVs( @Param("creatorID")Long creatorID,@Param("appID") Long appID);
+ List<ConfigKeyValue> selectAllAppKVs(@Param("creatorID") Long creatorID, @Param("appID") Long appID);
- void updateUserValue(@Param("value") String value,@Param("id") Long id);
+ void updateUserValue(@Param("value") String value, @Param("id") Long id);
ConfigKey selectKeyByKeyID(Long keyID);
+ List<ConfigKey> listKeyByCreatorAndAppName(@Param("creatorID") Long creatorID, @Param("appID") Long appID);
+
+ void insertCreator(String creator);
+
+ Long selectTreeIDByKeyID(Long keyID);
+
+ void insertKey(ConfigKey key);
+
+ void insertKeyTree(@Param("keyID") Long keyID, @Param("treeID") Long treeID);
+
+ ConfigTree selectTreeByAppIDAndName(@Param("appID") Long appID, @Param("treeName") String treeName);
+
}
diff --git a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/impl/ConfigMapper.xml b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/impl/ConfigMapper.xml
index 0737ae2..610f935 100644
--- a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/impl/ConfigMapper.xml
+++ b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/dao/impl/ConfigMapper.xml
@@ -24,6 +24,10 @@
id, `application_id`,`key_id`,`user_name`,`value`
</sql>
+ <sql id="key_List">
+ id, `key`,`description`,`name`,`application_id`,`default_value`,`validate_type`,`validate_range`,`is_hidden`,`is_advanced`,`level`,`unit`
+ </sql>
+
<select id="selectAppIDByAppName" resultType="Long">
SELECT bda.id from linkis_application bda WHERE bda.name=#{name}
</select>
@@ -113,4 +117,45 @@
SELECT * from linkis_config_key where id = #{keyID}
</select>
+ <select id="listKeyByCreatorAndAppName" resultType="com.webank.wedatasphere.linkis.configuration.entity.ConfigKey">
+ SELECT
+ `key`.*
+ FROM
+ linkis_config_tree tree,
+ linkis_config_key_tree kt,
+ linkis_config_key `key`
+ WHERE
+ tree.id = kt.tree_id
+ AND kt.key_id = `key`.id
+ AND `key`.application_id = #{creatorID}
+ AND tree.application_id = #{appID}
+ </select>
+
+ <insert id="insertCreator">
+ INSERT INTO linkis_application values(NULL,#{creator},NULL,NULL)
+ </insert>
+
+ <select id="selectTreeIDByKeyID" resultType="Long">
+ SELECT
+ kt.tree_id
+ FROM
+ linkis_config_key_tree kt
+ WHERE
+ kt.key_id = #{keyID}
+ </select>
+
+ <insert id="insertKey" useGeneratedKeys="true" keyProperty="id" parameterType ="com.webank.wedatasphere.linkis.configuration.entity.ConfigKey">
+ INSERT INTO linkis_config_key(<include refid="key_List"/>)
+ VALUES (#{id},#{key},#{description},
+ #{name},#{applicationID},#{defaultValue},#{validateType},#{validateRange},#{isAdvanced},#{isHidden},#{level},#{unit})
+ </insert>
+
+ <insert id="insertKeyTree">
+ INSERT INTO linkis_config_key_tree values(NULL,#{keyID},#{treeID})
+ </insert>
+
+ <select id="selectTreeByAppIDAndName" resultType="com.webank.wedatasphere.linkis.configuration.entity.ConfigTree">
+ SELECT * FROM linkis_config_tree WHERE `name` = #{treeName} AND `application_id` = #{appID}
+ </select>
+
</mapper>
\ No newline at end of file
diff --git a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKey.java b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKey.java
index e4af929..ca5dc19 100644
--- a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKey.java
+++ b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKey.java
@@ -32,7 +32,15 @@
private Boolean isAdvanced;
private Boolean isHidden;
private Integer level;
+ private String unit;
+ public String getUnit() {
+ return unit;
+ }
+
+ public void setUnit(String unit) {
+ this.unit = unit;
+ }
public Long getId() {
return id;
diff --git a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValue.java b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValue.java
index 5809306..387d0bc 100644
--- a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValue.java
+++ b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValue.java
@@ -36,6 +36,15 @@
private Long applicationID;
private String userName;
private String value;
+ private String unit;
+
+ public String getUnit() {
+ return unit;
+ }
+
+ public void setUnit(String unit) {
+ this.unit = unit;
+ }
public Long getKeyID() {
return keyID;
diff --git a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValueVO.java b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValueVO.java
index a3f0ab6..74cd232 100644
--- a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValueVO.java
+++ b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/entity/ConfigKeyValueVO.java
@@ -31,6 +31,33 @@
private String value;
private Integer level;
private String defaultValue;
+ private String validateType;
+ private String validateRange;
+ private String unit;
+
+ public String getUnit() {
+ return unit;
+ }
+
+ public void setUnit(String unit) {
+ this.unit = unit;
+ }
+
+ public String getValidateRange() {
+ return validateRange;
+ }
+
+ public void setValidateRange(String validateRange) {
+ this.validateRange = validateRange;
+ }
+
+ public String getValidateType() {
+ return validateType;
+ }
+
+ public void setValidateType(String validateType) {
+ this.validateType = validateType;
+ }
public Long getKeyID() {
return keyID;
diff --git a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/restful/api/ConfigurationRestfulApi.java
index 24bc038..ecb571c 100644
--- a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/restful/api/ConfigurationRestfulApi.java
+++ b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/restful/api/ConfigurationRestfulApi.java
@@ -20,7 +20,9 @@
import com.webank.wedatasphere.linkis.configuration.entity.ConfigKeyUser;
import com.webank.wedatasphere.linkis.configuration.entity.ConfigKeyValueVO;
import com.webank.wedatasphere.linkis.configuration.entity.ConfigTree;
+import com.webank.wedatasphere.linkis.configuration.exception.ConfigurationException;
import com.webank.wedatasphere.linkis.configuration.service.ConfigurationService;
+import com.webank.wedatasphere.linkis.configuration.util.ConfigurationConfiguration;
import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper;
import com.webank.wedatasphere.linkis.server.Message;
import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
@@ -28,6 +30,7 @@
import org.codehaus.jackson.map.ObjectMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
@@ -53,6 +56,61 @@
ObjectMapper mapper = new ObjectMapper();
@GET
+ @Path("/addKey")
+ public Response addKey(@Context HttpServletRequest req,
+ @QueryParam("appName") String appName,
+ @QueryParam("creator") String creator,
+ @QueryParam("token") String token,
+ @QueryParam("treeName") String treeName,
+ @QueryParam("keyJson") String keyJson) throws ConfigurationException {
+ String username = SecurityFilter.getLoginUsername(req);
+ if(StringUtils.isEmpty(appName) || StringUtils.isEmpty(creator) || StringUtils.isEmpty(token)){
+ throw new ConfigurationException("params cannot be empty!");
+ }
+ //todo检验token
+ if(!token.equals(ConfigurationConfiguration.COPYKEYTOKEN)){
+ throw new ConfigurationException("token is error");
+ }
+ List<ConfigKey> keys = configurationService.listKeyByCreatorAndAppName(creator,appName);
+ if(keys.isEmpty()){
+ //判断和copyKeyFromIDE相反,只允许在有key的情况下添加
+ throw new ConfigurationException(creator + ":" + appName + ",keys is empty ,cannot add key");
+ }
+ ConfigKey configKey = BDPJettyServerHelper.gson().fromJson(keyJson, ConfigKey.class);
+ // TODO: 2019/12/30 configKey参数校验
+ configurationService.addKey(creator,appName,treeName,configKey);
+ return Message.messageToResponse(Message.ok());
+ }
+
+
+ @GET
+ @Path("/copyKeyFromIDE")
+ public Response copyKeyFromIDE(@Context HttpServletRequest req,
+ @QueryParam("appName") String appName,
+ @QueryParam("creator") String creator,
+ @QueryParam("token") String token) throws ConfigurationException {
+ String username = SecurityFilter.getLoginUsername(req);
+ if(StringUtils.isEmpty(appName) || StringUtils.isEmpty(creator) || StringUtils.isEmpty(token)){
+ throw new ConfigurationException("params cannot be empty!");
+ }
+ //todo检验token
+ if(!token.equals(ConfigurationConfiguration.COPYKEYTOKEN)){
+ throw new ConfigurationException("token is error");
+ }
+ List<ConfigKey> keys = configurationService.listKeyByCreatorAndAppName(creator,appName);
+ if(!keys.isEmpty()){
+ throw new ConfigurationException(creator + ":" + appName + ",keys is no empty cannot copy key");
+ }
+ configurationService.insertCreator(creator);
+ List<ConfigKey> IDEkeys = configurationService.listKeyByCreatorAndAppName("IDE",appName);
+ if (IDEkeys.isEmpty()) {
+ throw new ConfigurationException("IDE:"+ appName + ",cannot find any key to copy");
+ }
+ IDEkeys.forEach(k ->configurationService.copyKeyFromIDE(k,creator,appName));
+ return Message.messageToResponse(Message.ok());
+ }
+
+ @GET
@Path("/getFullTreesByAppName")
public Response getFullTreesByAppName(@Context HttpServletRequest req, @QueryParam("appName") String appName, @QueryParam("creator") String creator) {
String username = SecurityFilter.getLoginUsername(req);
diff --git a/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/util/ConfigurationConfiguration.java b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/util/ConfigurationConfiguration.java
new file mode 100644
index 0000000..8ce5b0c
--- /dev/null
+++ b/publicService/configuration/src/main/java/com/webank/wedatasphere/linkis/configuration/util/ConfigurationConfiguration.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.configuration.util;
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars$;
+
+/**
+ * Created by patinousward on 2019/12/30.
+ */
+public class ConfigurationConfiguration {
+
+ public static final String COPYKEYTOKEN = CommonVars$.MODULE$.apply("wds.linkis.configuration.copykey.token","e1051-s").getValue();
+}
diff --git a/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/service/ConfigurationService.scala b/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/service/ConfigurationService.scala
index 5628c36..b692243 100644
--- a/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/service/ConfigurationService.scala
+++ b/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/service/ConfigurationService.scala
@@ -21,7 +21,7 @@
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.configuration.dao.ConfigMapper
-import com.webank.wedatasphere.linkis.configuration.entity._
+import com.webank.wedatasphere.linkis.configuration.entity.{ConfigKey, _}
import com.webank.wedatasphere.linkis.configuration.exception.ConfigurationException
import com.webank.wedatasphere.linkis.configuration.util.Constants
import com.webank.wedatasphere.linkis.configuration.validate.ValidatorManager
@@ -31,6 +31,7 @@
import org.springframework.beans.BeanUtils
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
+import org.springframework.transaction.annotation.Transactional
/**
* Created by allenlliu on 2019/4/8.
@@ -42,9 +43,44 @@
@Autowired private var validatorManager: ValidatorManager = _
+ @Transactional
+ def addKey(creator:String,appName:String,treeName:String,key: ConfigKey): Unit ={
+ val appID:Long = configMapper.selectAppIDByAppName(appName)
+ val creatorID: Long = configMapper.selectAppIDByAppName(creator)
+ val tree:ConfigTree = configMapper.selectTreeByAppIDAndName(appID,treeName)
+ key.setId(null)
+ key.setApplicationID(creatorID)
+ configMapper.insertKey(key)
+ configMapper.insertKeyTree(key.getId,tree.getId)
+ }
+
+ def insertCreator(creator:String): Unit ={
+ val creatorID: Long = configMapper.selectAppIDByAppName(creator)
+ if(creatorID == null) configMapper.insertCreator(creator) else warn(s"creator${creator} exists")
+ }
+
+ def listKeyByCreatorAndAppName(creator:String, appName:String):java.util.List[ConfigKey] ={
+ val creatorID: Long = configMapper.selectAppIDByAppName(creator)
+ val appID:Long = configMapper.selectAppIDByAppName(appName)
+ configMapper.listKeyByCreatorAndAppName(creatorID,appID)
+ }
+
+ @Transactional
+ def copyKeyFromIDE(key:ConfigKey,creator:String, appName:String) ={
+ val creatorID: Long = configMapper.selectAppIDByAppName(creator)
+ key.setApplicationID(creatorID)
+ val treeID = configMapper.selectTreeIDByKeyID(key.getId)
+ key.setId(null)
+ configMapper.insertKey(key)
+ configMapper.insertKeyTree(key.getId,treeID)
+ }
+
def updateUserValue(setting: ConfigKeyValueVO) = {
if (!StringUtils.isEmpty(setting.getValue)) {
+ if(!StringUtils.isEmpty(setting.getUnit) && !setting.getValue.contains(setting.getUnit) && !setting.getValue.contains(setting.getUnit.toLowerCase)){
+ setting.setValue(setting.getValue + setting.getUnit)
+ }
val key = configMapper.selectKeyByKeyID(setting.getKeyID)
info(s"Save parameter ${key.getKey} value ${setting.getValue} is not empty, enter checksum...(保存参数${key.getKey}值${setting.getValue}不为空，进入校验...)")
if (!validatorManager.getOrCreateValidator(key.getValidateType).validate(setting.getValue, key.getValidateRange)) {
@@ -83,6 +119,7 @@
vo.setValidateType(key.getValidateType)
vo.setValue(value.getValue)
vo.setValueID(value.getId)
+ vo.setUnit(key.getUnit)
vo
}
@@ -138,8 +175,8 @@
}
private def getValueByKeyId(keyID: Long, userName: String, appName: String): ConfigKeyUser = {
- val appID = configMapper.selectAppIDByAppName(appName);
- configMapper.selectValueByKeyId(keyID, userName, appID);
+ val appID = configMapper.selectAppIDByAppName(appName)
+ configMapper.selectValueByKeyId(keyID, userName, appID)
}
def queryAppConfigWithGlobal(userName: String, creator: String, appName: String, isMerge: Boolean): ResponseQueryConfig = {
diff --git a/publicService/jobhistory/pom.xml b/publicService/jobhistory/pom.xml
index 195f3cb..182ebfd 100644
--- a/publicService/jobhistory/pom.xml
+++ b/publicService/jobhistory/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-jobhistory</artifactId>
@@ -41,6 +41,12 @@
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-storage</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!--<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper-spring-boot-starter</artifactId>
diff --git a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/TaskMapper.java b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/TaskMapper.java
index 7d83124..338450c 100644
--- a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/TaskMapper.java
+++ b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/TaskMapper.java
@@ -33,8 +33,8 @@
void updateTask(QueryTask queryTask);
- List<QueryTask> search(@Param("taskID")Long taskID, @Param("umUser")String username, @Param("status")List<String> status,
- @Param("startDate")Date startDate, @Param("endDate")Date endDate,@Param("executeApplicationName")String executeApplicationName);
+ List<QueryTask> search(@Param("taskID") Long taskID, @Param("umUser") String username, @Param("status") List<String> status,
+ @Param("startDate") Date startDate, @Param("endDate") Date endDate, @Param("executeApplicationName") String executeApplicationName);
String selectTaskStatusForUpdate(Long taskID);
}
diff --git a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/impl/taskMapper.xml b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/impl/taskMapper.xml
index a5bc561..d06ba77 100644
--- a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/impl/taskMapper.xml
+++ b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/dao/impl/taskMapper.xml
@@ -28,7 +28,8 @@
<sql id="task_List">
id, `instance`, `exec_id`, `um_user`, `execution_code`, `progress`,`log_path`,
- `result_location`,`status`,`created_time`,`updated_time`,`run_type`,`err_code`,`err_desc`,`execute_application_name`,`request_application_name`,`script_path`,`params`,`engine_instance`
+ `result_location`,`status`,`created_time`,`updated_time`,`run_type`,`err_code`,`err_desc`,`execute_application_name`,`request_application_name`,`script_path`,`params`,`engine_instance`,
+ `engine_start_time`
</sql>
<insert id="insertTask" useGeneratedKeys="true" keyProperty="taskID" parameterType="com.webank.wedatasphere.linkis.jobhistory.entity.QueryTask">
@@ -37,7 +38,9 @@
#{umUser},#{executionCode},#{progress},
#{logPath},#{resultLocation},#{status},
#{createdTime},#{updatedTime},#{runType},
- #{errCode},#{errDesc},#{executeApplicationName},#{requestApplicationName},#{sourceJson},#{paramsJson},#{engineInstance})
+ #{errCode},#{errDesc},#{executeApplicationName},
+ #{requestApplicationName},#{sourceJson},#{paramsJson},
+ #{engineInstance},#{engineStartTime})
</insert>
<select id="selectTask" resultMap="taskMapper" parameterType="com.webank.wedatasphere.linkis.jobhistory.entity.QueryTask">
@@ -62,6 +65,7 @@
<if test="requestApplicationName != null">and request_application_name = #{requestApplicationName}</if>
<if test="sourceJson != null">and script_path = #{sourceJson}</if>
<if test="paramsJson != null">and params = #{paramsJson}</if>
+ <if test="engineStartTime != null">and engine_start_time = #{engineStartTime}</if>
</where>
</select>
@@ -99,6 +103,7 @@
<if test="sourceJson != null"> script_path = #{sourceJson},</if>
<if test="paramsJson != null"> params = #{paramsJson},</if>
<if test="engineInstance != null"> engine_instance = #{engineInstance},</if>
+ <if test="engineStartTime != null"> engine_start_time = #{engineStartTime},</if>
</trim>
WHERE id =#{taskID}
</update>
diff --git a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTask.java b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTask.java
index 6dee174..c989eb8 100644
--- a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTask.java
+++ b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTask.java
@@ -42,6 +42,14 @@
private String runType;
private String paramsJson;
private String sourceJson;
+ private Date engineStartTime;
+ public Date getEngineStartTime() {
+ return engineStartTime;
+ }
+
+ public void setEngineStartTime(Date engineStartTime) {
+ this.engineStartTime = engineStartTime;
+ }
public String getSourceJson() {
return sourceJson;
diff --git a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTaskVO.java b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTaskVO.java
index 2428f33..50d5318 100644
--- a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTaskVO.java
+++ b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/entity/QueryTaskVO.java
@@ -39,12 +39,33 @@
private String errDesc;
private String executeApplicationName;
private String requestApplicationName;
- private String scriptPath;
private String runType;
private String paramsJson;
private Long costTime;
private String strongerExecId;
private String sourceJson;
+ /**
+ * source字段:用来将sourceJson的value取出来进行拼接返回给前台展示
+ */
+ private String sourceTailor;
+
+ private Date engineStartTime;
+
+ public Date getEngineStartTime() {
+ return engineStartTime;
+ }
+
+ public void setEngineStartTime(Date engineStartTime) {
+ this.engineStartTime = engineStartTime;
+ }
+
+ public String getSourceTailor() {
+ return sourceTailor;
+ }
+
+ public void setSourceTailor(String sourceTailor) {
+ this.sourceTailor = sourceTailor;
+ }
public String getSourceJson() {
return sourceJson;
@@ -190,14 +211,6 @@
this.requestApplicationName = requestApplicationName;
}
- public String getScriptPath() {
- return scriptPath;
- }
-
- public void setScriptPath(String scriptPath) {
- this.scriptPath = scriptPath;
- }
-
public String getRunType() {
return runType;
}
diff --git a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/restful/api/QueryRestfulApi.java b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/restful/api/QueryRestfulApi.java
index 4be67b7..41e1af8 100644
--- a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/restful/api/QueryRestfulApi.java
+++ b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/restful/api/QueryRestfulApi.java
@@ -22,7 +22,6 @@
import com.webank.wedatasphere.linkis.jobhistory.entity.QueryTaskVO;
import com.webank.wedatasphere.linkis.jobhistory.exception.QueryException;
import com.webank.wedatasphere.linkis.jobhistory.service.QueryService;
-import com.webank.wedatasphere.linkis.jobhistory.util.QueryUtil;
import com.webank.wedatasphere.linkis.server.Message;
import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
import org.slf4j.Logger;
@@ -80,7 +79,6 @@
if (startDate != null && endDate == null){
endDate = System.currentTimeMillis();
}
- PageHelper.startPage(pageNow,pageSize);
Date sDate = null;
Date eDate = null;
if (startDate != null){
@@ -93,11 +91,18 @@
instance.add(Calendar.DAY_OF_MONTH,1);
eDate = instance.getTime();
}
- List<QueryTask> queryTasks = queryService.search(taskID,username,status, sDate,eDate,executeApplicationName);
+ List<QueryTask> queryTasks = null;
+ PageHelper.startPage(pageNow,pageSize);
+ try {
+ queryTasks = queryService.search(taskID,username,status, sDate,eDate,executeApplicationName);
+ }finally {
+ PageHelper.clearPage();
+ }
+
PageInfo<QueryTask> pageInfo = new PageInfo<>(queryTasks);
List<QueryTask> list = pageInfo.getList();
long total = pageInfo.getTotal();
- List<QueryTaskVO> vos = QueryUtil.getQueryVOList(list);
+ List<QueryTaskVO> vos = queryService.getQueryVOList(list);
return Message.messageToResponse(Message.ok().data("tasks", vos).data("totalPage",total));
}
}
diff --git a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/transitional/TransitionalQueryService.java b/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/transitional/TransitionalQueryService.java
deleted file mode 100644
index d86892e..0000000
--- a/publicService/jobhistory/src/main/java/com/webank/wedatasphere/linkis/jobhistory/transitional/TransitionalQueryService.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.jobhistory.transitional;
-
-import com.webank.wedatasphere.linkis.protocol.query.RequestUpdateTask;
-import com.webank.wedatasphere.linkis.protocol.query.ResponsePersist;
-import com.webank.wedatasphere.linkis.jobhistory.dao.TaskMapper;
-import com.webank.wedatasphere.linkis.jobhistory.exception.QueryException;
-import com.webank.wedatasphere.linkis.jobhistory.service.impl.QueryServiceImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.transaction.annotation.Transactional;
-
-import java.util.HashMap;
-
-/**
- * Created by johnnwang on 2019/6/6.
- */
-@Component
-public class TransitionalQueryService {
-
- private Logger logger = LoggerFactory.getLogger(this.getClass());
-
- @Autowired
- private TaskMapper taskMapper;
-
- @Autowired
- private QueryServiceImpl queryServiceImpl;
-
- @Transactional
- public ResponsePersist change(RequestUpdateTask requestUpdateTask) {
- ResponsePersist persist = new ResponsePersist();
- try {
- if (requestUpdateTask.getStatus() != null) {
- String oldStatus = taskMapper.selectTaskStatusForUpdate(requestUpdateTask.getTaskID());
- if (oldStatus != null && !shouldUpdate(oldStatus, requestUpdateTask.getStatus()))
- throw new QueryException(requestUpdateTask.getTaskID() + "The task state in the database is(数据库中的task状态为)：" + oldStatus + "The updated task status is(更新的task状态为)：" + requestUpdateTask.getStatus() + "Update failed!(更新失败)！");
- }
- taskMapper.updateTask(queryServiceImpl.requestPersistTaskTask2QueryTask(requestUpdateTask));
- HashMap<String, Object> map = new HashMap<>();
- map.put("taskID", requestUpdateTask.getTaskID());
- persist.setStatus(0);
- persist.setData(map);
- } catch (Exception e) {
- logger.error(e.getMessage());
- persist.setStatus(1);
- persist.setMsg(e.getMessage());
- }
- return persist;
- }
-
- private boolean shouldUpdate(String oldStatus, String newStatus) {
- return TaskStatus.valueOf(oldStatus).ordinal() <= TaskStatus.valueOf(newStatus).ordinal();
- }
-}
diff --git a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/conversions/TaskConversions.scala b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/conversions/TaskConversions.scala
index 07a89e6..bc370c2 100644
--- a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/conversions/TaskConversions.scala
+++ b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/conversions/TaskConversions.scala
@@ -23,6 +23,8 @@
import com.webank.wedatasphere.linkis.protocol.query.{RequestPersistTask, RequestQueryTask}
import com.webank.wedatasphere.linkis.protocol.utils.ZuulEntranceUtils
import com.webank.wedatasphere.linkis.jobhistory.entity.{QueryTask, QueryTaskVO}
+import com.webank.wedatasphere.linkis.jobhistory.transitional.TaskStatus
+import com.webank.wedatasphere.linkis.jobhistory.util.QueryUtils
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
import org.springframework.beans.BeanUtils
@@ -33,6 +35,7 @@
*/
object TaskConversions extends Logging{
+ @Deprecated
implicit def requestQueryTask2QueryTask(requestQueryTask: RequestQueryTask): QueryTask = {
val task: QueryTask = new QueryTask
BeanUtils.copyProperties(requestQueryTask, task)
@@ -44,6 +47,7 @@
}
implicit def queryTask2RequestPersistTask(queryTask: QueryTask): RequestPersistTask = {
+ QueryUtils.exchangeExecutionCode(queryTask)
val task = new RequestPersistTask
BeanUtils.copyProperties(queryTask, task)
task.setSource(BDPJettyServerHelper.gson.fromJson(queryTask.getSourceJson, classOf[java.util.HashMap[String, String]]))
@@ -51,6 +55,7 @@
task
}
+ @Deprecated
implicit def requestPersistTaskTask2QueryTask(requestPersistTask: RequestPersistTask): QueryTask = {
val task: QueryTask = new QueryTask
BeanUtils.copyProperties(requestPersistTask, task)
@@ -62,8 +67,18 @@
}
implicit def queryTask2QueryTaskVO(queryTask: QueryTask): QueryTaskVO = {
+ QueryUtils.exchangeExecutionCode(queryTask)
val taskVO = new QueryTaskVO
BeanUtils.copyProperties(queryTask, taskVO)
+ if(!StringUtils.isEmpty(taskVO.getSourceJson)){
+ Utils.tryCatch{
+ val source = BDPJettyServerHelper.gson.fromJson(taskVO.getSourceJson,classOf[util.Map[String,String]])
+ import scala.collection.JavaConversions._
+ taskVO.setSourceTailor(source.map(_._2).foldLeft("")(_ + _ + "-").stripSuffix("-"))
+ }{
+ case _ =>warn("sourceJson deserializae failed,this task may be the old data")
+ }
+ }
if (queryTask.getExecId() != null && queryTask.getExecuteApplicationName() != null && queryTask.getInstance() != null) {
taskVO.setStrongerExecId(ZuulEntranceUtils.generateExecID(queryTask.getExecId(),
queryTask.getExecuteApplicationName(), queryTask.getInstance(), queryTask.getRequestApplicationName))
@@ -71,19 +86,18 @@
val status = queryTask.getStatus()
val createdTime = queryTask.getCreatedTime()
val updatedTime = queryTask.getUpdatedTime()
- if ("Succeed".equalsIgnoreCase(status) || "Failed".equalsIgnoreCase(status) || "Cancelled".equalsIgnoreCase(status)) {
- if (createdTime != null && updatedTime != null) {
- taskVO.setCostTime(queryTask.getUpdatedTime().getTime() - queryTask.getCreatedTime().getTime());
- } else {
- taskVO.setCostTime(null)
- }
- } else {
- if (createdTime != null) {
- taskVO.setCostTime(System.currentTimeMillis() - queryTask.getCreatedTime().getTime());
- } else {
- taskVO.setCostTime(null)
- }
+ if (isJobFinished(status) && createdTime != null && updatedTime != null) {
+ taskVO.setCostTime(queryTask.getUpdatedTime().getTime() - queryTask.getCreatedTime().getTime());
+ } else if (createdTime != null) {
+ taskVO.setCostTime(System.currentTimeMillis() - queryTask.getCreatedTime().getTime());
}
taskVO
}
-}
+
+ def isJobFinished(status: String): Boolean = {
+ TaskStatus.Succeed.toString.equals(status) ||
+ TaskStatus.Failed.toString.equals(status) ||
+ TaskStatus.Cancelled.toString.equals(status) ||
+ TaskStatus.Timeout.toString.equals(status)
+ }
+}
\ No newline at end of file
diff --git a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/QueryService.java b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/QueryService.java
index 60e247a..a4ec6d0 100644
--- a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/QueryService.java
+++ b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/QueryService.java
@@ -34,7 +34,9 @@
ResponsePersist query(RequestQueryTask requestQueryTask);
- QueryTaskVO getTaskByID(Long taskID,String userName);
+ QueryTaskVO getTaskByID(Long taskID, String userName);
List<QueryTask> search(Long taskID, String username, String status, Date sDate, Date eDate, String executeApplicationName);
+
+ List<QueryTaskVO> getQueryVOList(List<QueryTask> list);
}
diff --git a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/impl/QueryServiceImpl.scala b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/impl/QueryServiceImpl.scala
index ba170d4..95af6e9 100644
--- a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/impl/QueryServiceImpl.scala
+++ b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/service/impl/QueryServiceImpl.scala
@@ -16,33 +16,34 @@
package com.webank.wedatasphere.linkis.jobhistory.service.impl
-import java.lang.Long
import java.util
import java.util.Date
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.jobhistory.dao.TaskMapper
import com.webank.wedatasphere.linkis.jobhistory.entity.{QueryTask, QueryTaskVO}
+import com.webank.wedatasphere.linkis.jobhistory.exception.QueryException
import com.webank.wedatasphere.linkis.jobhistory.service.QueryService
-import com.webank.wedatasphere.linkis.jobhistory.transitional.TransitionalQueryService
+import com.webank.wedatasphere.linkis.jobhistory.transitional.TaskStatus
+import com.webank.wedatasphere.linkis.jobhistory.util.QueryUtils
import com.webank.wedatasphere.linkis.protocol.query._
import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper
import org.springframework.beans.BeanUtils
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
+import org.springframework.transaction.annotation.Transactional
/**
- * Created by johnnwang on 2019/2/25.
- */
+ * Created by johnnwang on 2019/2/25.
+ */
@Service
class QueryServiceImpl extends QueryService with Logging {
@Autowired
private var taskMapper: TaskMapper = _
- @Autowired
- private var transitionalQueryService:TransitionalQueryService = _
override def add(requestInsertTask: RequestInsertTask): ResponsePersist = {
info("Insert data into the database(往数据库中插入数据)：" + requestInsertTask.toString)
+ QueryUtils.storeExecutionCode(requestInsertTask)
val persist = new ResponsePersist
Utils.tryCatch {
val queryTask = requestPersistTaskTask2QueryTask(requestInsertTask)
@@ -54,18 +55,47 @@
} {
case e: Exception =>
error(e.getMessage)
+ persist.setStatus(1)
+ persist.setMsg(e.getMessage)
+ }
+ persist
+ }
+
+ @Transactional
+ override def change(requestUpdateTask: RequestUpdateTask): ResponsePersist = {
+ val executionCode = requestUpdateTask.getExecutionCode
+ requestUpdateTask.setExecutionCode(null)
+ info("Update data to the database(往数据库中更新数据)：" + requestUpdateTask.toString)
+ val persist = new ResponsePersist
+ Utils.tryCatch {
+ if (requestUpdateTask.getErrDesc != null) {
+ if (requestUpdateTask.getErrDesc.length > 256) {
+ info(s"errorDesc is too long,we will cut some message")
+ requestUpdateTask.setErrDesc(requestUpdateTask.getErrDesc.substring(0, 256))
+ info(s"${requestUpdateTask.getErrDesc}")
+ }
+ }
+ if (requestUpdateTask.getStatus != null) {
+ val oldStatus: String = taskMapper.selectTaskStatusForUpdate(requestUpdateTask.getTaskID)
+ if (oldStatus != null && !shouldUpdate(oldStatus, requestUpdateTask.getStatus))
+ throw new QueryException(s"${requestUpdateTask.getTaskID}数据库中的task状态为：${oldStatus}更新的task状态为：${requestUpdateTask.getStatus}更新失败！")
+ }
+ taskMapper.updateTask(requestPersistTaskTask2QueryTask(requestUpdateTask))
+ val map = new util.HashMap[String, Object]
+ map.put("taskID", requestUpdateTask.getTaskID)
+ persist.setStatus(0)
+ persist.setData(map)
+ } {
+ case e: Exception =>
+ error(e.getMessage)
persist.setStatus(1);
persist.setMsg(e.getMessage);
}
persist
}
- override def change(requestUpdateTask: RequestUpdateTask): ResponsePersist = {
- info("Update data to the database(往数据库中更新数据)：" + requestUpdateTask.toString)
- transitionalQueryService.change(requestUpdateTask)
- }
-
override def query(requestQueryTask: RequestQueryTask): ResponsePersist = {
+ info("查询历史task：" + requestQueryTask.toString)
val persist = new ResponsePersist
Utils.tryCatch {
val task = taskMapper.selectTask(requestPersistTaskTask2QueryTask(requestQueryTask))
@@ -90,34 +120,43 @@
tasks
}
- def requestPersistTaskTask2QueryTask(requestPersistTask: RequestPersistTask): QueryTask = {
+ private def requestPersistTaskTask2QueryTask(requestPersistTask: RequestPersistTask): QueryTask = {
val task: QueryTask = new QueryTask
BeanUtils.copyProperties(requestPersistTask, task)
- if(requestPersistTask.getSource != null)
+ if (requestPersistTask.getSource != null)
task.setSourceJson(BDPJettyServerHelper.gson.toJson(requestPersistTask.getSource))
if (requestPersistTask.getParams != null)
task.setParamsJson(BDPJettyServerHelper.gson.toJson(requestPersistTask.getParams))
- else
- task.setParamsJson(null)
task
}
- override def getTaskByID(taskID: Long,userName:String): QueryTaskVO = {
+ override def getTaskByID(taskID: java.lang.Long, userName: String): QueryTaskVO = {
val task = new QueryTask
task.setTaskID(taskID)
task.setUmUser(userName)
val taskR = taskMapper.selectTask(task)
- if (taskR.size() > 0) {
- import com.webank.wedatasphere.linkis.jobhistory.conversions.TaskConversions.queryTask2QueryTaskVO
- taskR.get(0)
- } else {
- null
- }
+ import com.webank.wedatasphere.linkis.jobhistory.conversions.TaskConversions.queryTask2QueryTaskVO
+
+ import scala.collection.JavaConversions._
+ if (taskR.isEmpty) null else taskR(0)
}
- override def search(taskID: Long, username: String, status: String, sDate: Date, eDate: Date, executeApplicationName: String): util.List[QueryTask] = {
+ override def search(taskID: java.lang.Long, username: String, status: String, sDate: Date, eDate: Date, executeApplicationName: String): util.List[QueryTask] = {
import scala.collection.JavaConversions._
val split: util.List[String] = if (status != null) status.split(",").toList else null
taskMapper.search(taskID, username, split, sDate, eDate, executeApplicationName)
}
+
+ def getQueryVOList(list: java.util.List[QueryTask]): java.util.List[QueryTaskVO] = {
+ val ovs = new util.ArrayList[QueryTaskVO]
+ import scala.collection.JavaConversions._
+ list.foreach(f => {
+ import com.webank.wedatasphere.linkis.jobhistory.conversions.TaskConversions.queryTask2QueryTaskVO
+ ovs.add(f)
+ })
+ ovs
+ }
+
+ private def shouldUpdate(oldStatus: String, newStatus: String): Boolean = TaskStatus.valueOf(oldStatus).ordinal <= TaskStatus.valueOf(newStatus).ordinal
}
+
diff --git a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/util/QueryUtil.scala b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/util/QueryUtil.scala
deleted file mode 100644
index c99a22d..0000000
--- a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/util/QueryUtil.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.jobhistory.util
-
-import java.util
-import scala.collection.JavaConversions._
-import com.webank.wedatasphere.linkis.jobhistory.entity.{QueryTask, QueryTaskVO}
-
-/**
- * Created by johnnwang on 2019/2/25.
- */
-object QueryUtil {
- def getQueryVOList(list:java.util.List[QueryTask]):java.util.List[QueryTaskVO] ={
- val ovs = new util.ArrayList[QueryTaskVO]
- list.foreach(f =>{
- import com.webank.wedatasphere.linkis.jobhistory.conversions.TaskConversions.queryTask2QueryTaskVO
- ovs.add(f)
- })
- ovs
- }
-}
diff --git a/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/util/QueryUtils.scala b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/util/QueryUtils.scala
new file mode 100644
index 0000000..789e4ce
--- /dev/null
+++ b/publicService/jobhistory/src/main/scala/com/webank/wedatasphere/linkis/jobhistory/util/QueryUtils.scala
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.jobhistory.util
+
+import java.io.{InputStream, OutputStream}
+import java.util.Date
+
+import com.webank.wedatasphere.linkis.common.conf.CommonVars
+import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.jobhistory.entity.QueryTask
+import com.webank.wedatasphere.linkis.protocol.query.RequestInsertTask
+import com.webank.wedatasphere.linkis.storage.FSFactory
+import com.webank.wedatasphere.linkis.storage.fs.FileSystem
+import com.webank.wedatasphere.linkis.storage.utils.{FileSystemUtils, StorageUtils}
+import org.apache.commons.io.IOUtils
+import org.apache.commons.lang.time.DateFormatUtils
+
+/**
+ * Created by patinousward on 2019/2/25.
+ */
+object QueryUtils extends Logging {
+
+ private val CODE_STORE_PREFIX = CommonVars("bdp.dataworkcloud.query.store.prefix", "hdfs:///tmp/bdp-ide/")
+ private val CODE_STORE_SUFFIX = CommonVars("bdp.dataworkcloud.query.store.suffix", "")
+ private val CHARSET = "utf-8"
+ private val CODE_SPLIT = ";"
+ private val LENGTH_SPLIT = "#"
+
+ def storeExecutionCode(requestInsertTask: RequestInsertTask): Unit = {
+ if (requestInsertTask.getExecutionCode.length < 60000) return
+ val user: String = requestInsertTask.getUmUser
+ val path: String = getCodeStorePath(user)
+ val fsPath: FsPath = new FsPath(path)
+ val fileSystem = FSFactory.getFsByProxyUser(fsPath, user).asInstanceOf[FileSystem]
+ fileSystem.init(null)
+ var os: OutputStream = null
+ var position = 0L
+ val codeBytes = requestInsertTask.getExecutionCode.getBytes(CHARSET)
+ path.intern() synchronized {
+ Utils.tryFinally {
+ if (!fileSystem.exists(fsPath)) FileSystemUtils.createNewFile(fsPath, user, true)
+ os = fileSystem.write(fsPath, false)
+ position = fileSystem.get(path).getLength
+ IOUtils.write(codeBytes, os)
+ } {
+ IOUtils.closeQuietly(os)
+ if (fileSystem != null) fileSystem.close()
+ }
+ }
+ val length = codeBytes.length
+ requestInsertTask.setExecutionCode(path + CODE_SPLIT + position + LENGTH_SPLIT + length)
+ }
+
+ def exchangeExecutionCode(queryTask: QueryTask): Unit = {
+ import scala.util.control.Breaks._
+ if (queryTask.getExecutionCode == null || !queryTask.getExecutionCode.startsWith(StorageUtils.HDFS_SCHEMA)) return
+ val codePath = queryTask.getExecutionCode
+ val path = codePath.substring(0, codePath.lastIndexOf(CODE_SPLIT))
+ val codeInfo = codePath.substring(codePath.lastIndexOf(CODE_SPLIT) + 1)
+ val infos: Array[String] = codeInfo.split(LENGTH_SPLIT)
+ val position = infos(0).toInt
+ var lengthLeft = infos(1).toInt
+ val tub = new Array[Byte](1024)
+ val executionCode: StringBuilder = new StringBuilder
+ val fsPath: FsPath = new FsPath(path)
+ val fileSystem = FSFactory.getFsByProxyUser(fsPath, queryTask.getUmUser).asInstanceOf[FileSystem]
+ fileSystem.init(null)
+ var is: InputStream = null
+ if (!fileSystem.exists(fsPath)) return
+ Utils.tryFinally {
+ is = fileSystem.read(fsPath)
+ if (position > 0) is.skip(position)
+ breakable {
+ while (lengthLeft > 0) {
+ val readed = is.read(tub)
+ val useful = Math.min(readed, lengthLeft)
+ if (useful < 0) break()
+ lengthLeft -= useful
+ executionCode.append(new String(tub, 0, useful, CHARSET))
+ }
+ }
+ } {
+ if (fileSystem != null) fileSystem.close()
+ IOUtils.closeQuietly(is)
+ }
+ queryTask.setExecutionCode(executionCode.toString())
+ }
+
+ private def getCodeStorePath(user: String): String = {
+ val date: String = DateFormatUtils.format(new Date, "yyyyMMdd")
+ s"${CODE_STORE_PREFIX.getValue}${user}${CODE_STORE_SUFFIX.getValue}/executionCode/${date}/_scripts"
+ }
+}
diff --git a/publicService/pom.xml b/publicService/pom.xml
index 5168a70..53617f2 100644
--- a/publicService/pom.xml
+++ b/publicService/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/publicService/udf/pom.xml b/publicService/udf/pom.xml
index 137c5ea..5320cfe 100644
--- a/publicService/udf/pom.xml
+++ b/publicService/udf/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/publicService/variable/pom.xml b/publicService/variable/pom.xml
index 16b2a48..97c38b2 100644
--- a/publicService/variable/pom.xml
+++ b/publicService/variable/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-variable</artifactId>
diff --git a/publicService/workspace/client/workspace-httpclient/pom.xml b/publicService/workspace/client/workspace-httpclient/pom.xml
index ec44637..967e45f 100644
--- a/publicService/workspace/client/workspace-httpclient/pom.xml
+++ b/publicService/workspace/client/workspace-httpclient/pom.xml
@@ -7,7 +7,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-workspace-httpclient</artifactId>
diff --git a/publicService/workspace/pom.xml b/publicService/workspace/pom.xml
index 3d93752..8a5b98d 100644
--- a/publicService/workspace/pom.xml
+++ b/publicService/workspace/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-workspace</artifactId>
<packaging>jar</packaging>
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/annotation/WorkspaceChooserBeanAnnotation.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/annotation/WorkspaceChooserBeanAnnotation.java
deleted file mode 100644
index f06f323..0000000
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/annotation/WorkspaceChooserBeanAnnotation.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * Created by johnnwang on 2019/4/15.
- */
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = WorkspaceChooserBeanAnnotation.BEAN_NAME)
-@Component(value = WorkspaceChooserBeanAnnotation.BEAN_NAME)
-public @interface WorkspaceChooserBeanAnnotation {
- String BEAN_NAME = "workspaceReceiverChooser";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface WorkspaceChooserAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
-
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/annotation/WorkspaceRPCServiceBeanAnnotation.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/annotation/WorkspaceRPCServiceBeanAnnotation.java
deleted file mode 100644
index d7de26d..0000000
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/annotation/WorkspaceRPCServiceBeanAnnotation.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * Created by johnnwang on 2019/4/15.
- */
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = WorkspaceRPCServiceBeanAnnotation.BEAN_NAME)
-@Component(value = WorkspaceRPCServiceBeanAnnotation.BEAN_NAME)
-public @interface WorkspaceRPCServiceBeanAnnotation {
- String BEAN_NAME = "workspaceRPCService";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface WorkspaceRPCServiceAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
\ No newline at end of file
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java
index 88fc581..6b5670e 100644
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java
@@ -19,17 +19,26 @@
import com.webank.wedatasphere.linkis.common.conf.CommonVars;
import com.webank.wedatasphere.linkis.common.conf.CommonVars$;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
/**
* Created by johnnwang on 2018/11/9.
*/
public class WorkSpaceConfiguration {
- public static final CommonVars LOCAL_USER_ROOT_PATH = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.localuserrootpath","file:///tmp/linkis/");
- public static final CommonVars HDFS_USER_ROOT_PATH_PREFIX = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix","hdfs:///tmp/");
- public static final CommonVars HDFS_USER_ROOT_PATH_SUFFIX = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.hdfsuserrootpath.suffix","/linkis/");
- public static final CommonVars<Integer> RESULT_SET_DOWNLOAD_MAX_SIZE = CommonVars$.MODULE$.apply("wds.linkis.workspace.resultset.download.maxsize",5000);
- public static final CommonVars<Boolean> RESULT_SET_DOWNLOAD_IS_LIMIT = CommonVars$.MODULE$.apply("wds.linkis.workspace.resultset.download.is.limit",true);
- public static final CommonVars<Integer> RESULT_SET_DOWNLOAD_MAX_SIZE_CSV = CommonVars$.MODULE$.apply("wds.linkis.workspace.resultset.download.maxsize.csv",5000);
- public static final CommonVars<Integer> RESULT_SET_DOWNLOAD_MAX_SIZE_EXCEL = CommonVars$.MODULE$.apply("wds.linkis.workspace.resultset.download.maxsize.excel",5000);
+ public static final CommonVars<String> LOCAL_USER_ROOT_PATH = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.localuserrootpath", "file:///tmp/linkis/");
+ public static final CommonVars<String> HDFS_USER_ROOT_PATH_PREFIX = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix", "hdfs:///tmp/");
+ public static final CommonVars<String> HDFS_USER_ROOT_PATH_SUFFIX = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.hdfsuserrootpath.suffix", "/linkis/");
+ public static final CommonVars<Boolean> RESULT_SET_DOWNLOAD_IS_LIMIT = CommonVars$.MODULE$.apply("wds.linkis.workspace.resultset.download.is.limit", true);
+ public static final CommonVars<Integer> RESULT_SET_DOWNLOAD_MAX_SIZE_CSV = CommonVars$.MODULE$.apply("wds.linkis.workspace.resultset.download.maxsize.csv", 5000);
+ public static final CommonVars<Integer> RESULT_SET_DOWNLOAD_MAX_SIZE_EXCEL = CommonVars$.MODULE$.apply("wds.linkis.workspace.resultset.download.maxsize.excel", 5000);
+ public static final CommonVars<Long> FILESYSTEM_GET_TIMEOUT = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.get.timeout", 2000L);
+ public static final CommonVars<Integer> FILESYSTEM_FS_THREAD_NUM = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.thread.num", 10);
+ public static final CommonVars<Integer> FILESYSTEM_FS_THREAD_CACHE = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.thread.cache", 1000);
public static final CommonVars<Boolean> FILESYSTEM_PATH_CHECK_TRIGGER = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.path.check", false);
+ public static final ExecutorService executorService =
+ new ThreadPoolExecutor(FILESYSTEM_FS_THREAD_NUM.getValue(), FILESYSTEM_FS_THREAD_NUM.getValue(), 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(FILESYSTEM_FS_THREAD_CACHE.getValue()));
}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/constant/WorkSpaceConstants.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/constant/WorkSpaceConstants.java
new file mode 100644
index 0000000..5246253
--- /dev/null
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/constant/WorkSpaceConstants.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.filesystem.constant;
+
+/**
+ * Created by patinousward on 2018/10/17.
+ */
+public class WorkSpaceConstants {
+ public static final String XLSX_RESPONSE_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
+ public static final String DEFAULT_DATE_TYPE = "yyyy-MM-dd HH:mm:ss";
+ public static final String LOCAL_RETURN_TYPE = "Local";
+ public static final String BLANK = "BLANK";
+}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/entity/LogLevel.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/entity/LogLevel.java
new file mode 100644
index 0000000..6db4a09
--- /dev/null
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/entity/LogLevel.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.filesystem.entity;
+
+import com.webank.wedatasphere.linkis.filesystem.util.WorkspaceUtil;
+
+public class LogLevel {
+
+ private Type type;
+
+ public LogLevel(Type type) {
+ this.type = type;
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ public void setType(Type type) {
+ this.type = type;
+ }
+
+ public enum Type {
+ ERROR(WorkspaceUtil.errorReg), WARN(WorkspaceUtil.warnReg), INFO(WorkspaceUtil.infoReg), ALL(WorkspaceUtil.allReg);
+ private String reg;
+
+ Type(String reg) {
+ this.reg = reg;
+ }
+
+ public String getReg() {
+ return this.reg;
+ }
+ }
+}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/exception/WorkSpaceException.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/exception/WorkSpaceException.java
index eea0abc..5ec3b12 100644
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/exception/WorkSpaceException.java
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/exception/WorkSpaceException.java
@@ -16,12 +16,18 @@
package com.webank.wedatasphere.linkis.filesystem.exception;
+import com.webank.wedatasphere.linkis.common.exception.ErrorException;
+
/**
* Created by johnnwang on 2018/10/30.
*/
-public class WorkSpaceException extends Exception {
+public class WorkSpaceException extends ErrorException {
- public WorkSpaceException(String msg){
- super(msg);
+ public WorkSpaceException(int errCode, String desc) {
+ super(errCode, desc);
+ }
+
+ public WorkSpaceException(int errCode, String desc, String ip, int port, String serviceKind) {
+ super(errCode, desc, ip, port, serviceKind);
}
}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/exception/WorkspaceExceptionManager.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/exception/WorkspaceExceptionManager.java
new file mode 100644
index 0000000..5b348f3
--- /dev/null
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/exception/WorkspaceExceptionManager.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.filesystem.exception;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created by patinousward on 2020/1/16.
+ */
+public class WorkspaceExceptionManager {
+
+ private static Map<String, String> desc = new HashMap<String, String>(64) {
+ {
+ put("80001", "Requesting IO-Engine to initialize fileSystem failed!(请求IO-Engine初始化fileSystem失败！)");
+ put("80002", "The user has obtained the filesystem for more than %ds. Please contact the administrator.（用户获取filesystem的时间超过%ds，请联系管理员）");
+ put("80003", "User local root directory does not exist, please contact administrator to add（用户本地根目录不存在,请联系管理员添加)");
+ put("80004", "path:(路径：)%sIs empty!(为空！)");
+ put("80005", "The created folder name is duplicated!(创建的文件夹名重复!)");
+ put("80006", "The file name created is duplicated!(创建的文件名重复!)");
+ put("80007", "The renamed name is repeated!(重命名的名字重复!)");
+ put("80008", "The deleted file or folder does not exist!(删除的文件or文件夹不存在!)");
+ put("80009", "This user does not have permission to delete this file or folder!(该用户无权删除此文件或文件夹！)");
+ put("80010", "The user does not have permission to view the contents of the directory(该用户无权限查看该目录的内容)");
+ put("80011", "The downloaded file does not exist!(下载的文件不存在!)");
+ put("80012", "This user has no permission to read this file!");
+ put("80013", "file does not exist!(文件不存在！)");
+ put("80014", "The user has no permission to modify the contents of this file and cannot save it!(该用户无权限对此文件内容进行修改，无法保存！)");
+ put("80015", "unsupported resultset output type");
+ put("80016", "The file content is empty and cannot be imported!(文件内容为空，不能进行导入操作！)");
+ put("80017", "The header of the file has no qualifiers. Do not check the first behavior header or set no qualifier!(该文件的表头没有限定符，请勿勾选首行为表头或者设置无限定符！)");
+ put("80018", "This user has no permission to read this log!(该用户无权限读取此日志！)");
+ put("80019", "scriptContent is empty,this is normal!");
+ put("80021", "上传失败");
+ put("80022", "更新失败");
+ put("80023", "下载失败");
+ put("80024", "非table类型的结果集不能下载为excel");
+ put("80028", "the path exist special char");
+ put("80029", "empty dir!");
+ put("80030", "illegal str %s");
+ }
+ };
+
+ public static WorkSpaceException createException(int errorCode, Object... format) throws WorkSpaceException {
+ return new WorkSpaceException(errorCode, String.format(desc.get(String.valueOf(errorCode)), format));
+ }
+
+}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/BMLFsRestfulApi.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/BMLFsRestfulApi.java
index 3ba93ab..026f891 100644
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/BMLFsRestfulApi.java
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/BMLFsRestfulApi.java
@@ -16,16 +16,18 @@
package com.webank.wedatasphere.linkis.filesystem.restful.api;
import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.common.io.FsPath;
import com.webank.wedatasphere.linkis.filesystem.bml.BMLHelper;
-import com.webank.wedatasphere.linkis.filesystem.bml.BMLScriptReader;
-import com.webank.wedatasphere.linkis.filesystem.bml.BMLScriptWriter;
import com.webank.wedatasphere.linkis.filesystem.exception.WorkSpaceException;
+import com.webank.wedatasphere.linkis.filesystem.exception.WorkspaceExceptionManager;
import com.webank.wedatasphere.linkis.server.Message;
import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
-import com.webank.wedatasphere.linkis.storage.script.ScriptMetaData;
-import com.webank.wedatasphere.linkis.storage.script.ScriptRecord;
-import com.webank.wedatasphere.linkis.storage.script.Variable;
-import com.webank.wedatasphere.linkis.storage.script.VariableParser;
+import com.webank.wedatasphere.linkis.storage.script.*;
+import com.webank.wedatasphere.linkis.storage.script.writer.StorageScriptFsWriter;
+import com.webank.wedatasphere.linkis.storage.source.FileSource;
+import com.webank.wedatasphere.linkis.storage.source.FileSource$;
+import org.apache.commons.math3.util.Pair;
+import org.apache.http.Consts;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
@@ -38,18 +40,18 @@
import javax.ws.rs.core.Response;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.stream.Collectors;
+
/**
* Created by patinousward
-*/
+ */
@Produces(MediaType.APPLICATION_JSON)
@Consumes({MediaType.APPLICATION_JSON})
@Component
@Path("filesystem")
public class BMLFsRestfulApi {
+
@Autowired
BMLHelper bmlHelper;
@@ -58,58 +60,96 @@
public Response openScriptFromBML(@Context HttpServletRequest req,
@QueryParam("resourceId") String resourceId,
@QueryParam("version") String version,
- @QueryParam("fileName") String fileName,String userNameRpc) throws IOException, WorkSpaceException {
- String userName = req == null?userNameRpc: SecurityFilter.getLoginUsername(req);
- Map<String, Object> query = bmlHelper.query(userName,resourceId,version);
- if(fileName ==null) fileName = "test_faker.sql"; // TODO: 2019/5/31 通过物料库获取 文件类型
- BMLScriptReader reader = BMLScriptReader.getBMLScriptReader((InputStream) query.get("stream"), fileName);
- ScriptMetaData metaData = (ScriptMetaData) reader.getMetaData();
- Map<String, Object> params = VariableParser.getMap(metaData.getMetaData());
- StringBuilder scriptContent = new StringBuilder();
- while (reader.hasNext()){
- ScriptRecord record = (ScriptRecord) reader.getRecord();
- scriptContent.append(record.getLine() +"\n");
+ @QueryParam("creator") String creator,
+ @QueryParam("projectName") String projectName,
+ @DefaultValue("test.sql") @QueryParam("fileName") String fileName) throws IOException, WorkSpaceException {
+ String userName = SecurityFilter.getLoginUsername(req);
+ Map<String, Object> query = bmlHelper.query(userName, resourceId, version);
+ InputStream inputStream = (InputStream) query.get("stream");
+ try (FileSource fileSource = FileSource$.MODULE$.create(new FsPath(fileName), inputStream)) {
+ Pair<Object, ArrayList<String[]>> collect = fileSource.collect();
+ Message message;
+ try {
+ message = new Gson().fromJson(collect.getSecond().get(0)[0], Message.class);
+ if (message == null) throw WorkspaceExceptionManager.createException(80019);
+ } catch (Exception e) {
+ return Message.messageToResponse(Message.ok().data("scriptContent", collect.getSecond().get(0)[0]).data("metadata", collect.getFirst()));
+ }
+ if (message.getStatus() != 0) {
+ throw new WorkSpaceException(80020, message.getMessage());
+ }
+ return Message.messageToResponse(Message.ok().data("scriptContent", collect.getSecond().get(0)[0]).data("metadata", collect.getFirst()));
}
- Message message = null;
- try {
- message = new Gson().fromJson(scriptContent.toString(), Message.class);
- }catch (Exception e){
- return Message.messageToResponse(Message.ok().data("scriptContent",scriptContent.toString()).data("metadata",params));
- }
- if(message.getStatus() != 0){
- throw new WorkSpaceException(message.getMessage());
- }
- return Message.messageToResponse(Message.ok().data("scriptContent",scriptContent.toString()).data("metadata",params));
}
+
+ @GET
+ @Path("/product/openScriptFromBML")
+ public Response openScriptFromProductBML(@Context HttpServletRequest req,
+ @QueryParam("resourceId") String resourceId,
+ @QueryParam("version") String version,
+ @QueryParam("creator") String creator,
+ @DefaultValue("test.sql") @QueryParam("fileName") String fileName) throws IOException, WorkSpaceException {
+ String userName = SecurityFilter.getLoginUsername(req);
+ if (!StringUtils.isEmpty(creator)){
+ userName = creator;
+ }
+ Map<String, Object> query = bmlHelper.query(userName, resourceId, version);
+ InputStream inputStream = (InputStream) query.get("stream");
+ try (FileSource fileSource = FileSource$.MODULE$.create(new FsPath(fileName), inputStream)) {
+ Pair<Object, ArrayList<String[]>> collect = fileSource.collect();
+ Message message;
+ try {
+ message = new Gson().fromJson(collect.getSecond().get(0)[0], Message.class);
+ if (message == null) {
+ throw WorkspaceExceptionManager.createException(80019);
+ }
+ } catch (Exception e) {
+ return Message.messageToResponse(Message.ok().data("scriptContent", collect.getSecond().get(0)[0]).data("metadata", collect.getFirst()));
+ }
+ if (message.getStatus() != 0) {
+ throw new WorkSpaceException(80020, message.getMessage());
+ }
+ return Message.messageToResponse(Message.ok().data("scriptContent", collect.getSecond().get(0)[0]).data("metadata", collect.getFirst()));
+ }
+ }
+
+
+
@POST
@Path("/saveScriptToBML")
public Response saveScriptToBML(@Context HttpServletRequest req, @RequestBody Map<String, Object> json) throws IOException {
- String userName = req == null?(String) json.get("userName"): SecurityFilter.getLoginUsername(req);
+ String userName = SecurityFilter.getLoginUsername(req);
String scriptContent = (String) json.get("scriptContent");
- Map<String, Object> params = (Map<String, Object>)json.get("metadata");
+ Map<String, Object> params = (Map<String, Object>) json.get("metadata");
String fileName = (String) json.get("fileName");
String resourceId = (String) json.get("resourceId");
- BMLScriptWriter writer = BMLScriptWriter.getBMLScriptWriter(fileName);
+ String creator = (String)json.get("creator");
+ String projectName = (String)json.get("projectName");
+ ScriptFsWriter writer = StorageScriptFsWriter.getScriptFsWriter(new FsPath(fileName), Consts.UTF_8.toString(), null);
Variable[] v = VariableParser.getVariables(params);
List<Variable> variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.value())).collect(Collectors.toList());
writer.addMetaData(new ScriptMetaData(variableList.toArray(new Variable[0])));
writer.addRecord(new ScriptRecord(scriptContent));
- InputStream inputStream = writer.getInputStream();
- String version=null;
- if(resourceId == null){
- // TODO: 2019/5/28 新增文件
- Map<String, Object> bmlResponse = bmlHelper.upload(userName, inputStream, fileName);
- resourceId = bmlResponse.get("resourceId").toString();
- version = bmlResponse.get("version").toString();
- }else {
- // TODO: 2019/5/28 更新文件
- Map<String, Object> bmlResponse = bmlHelper.update(userName, resourceId, inputStream);
- resourceId = bmlResponse.get("resourceId").toString();
- version = bmlResponse.get("version").toString();
+ try (InputStream inputStream = writer.getInputStream()) {
+ String version;
+ if (resourceId == null) {
+ // 新增文件
+ Map<String, Object> bmlResponse = new HashMap<>();
+ if (!StringUtils.isEmpty(projectName)) {
+ bmlResponse = bmlHelper.upload(userName, inputStream, fileName, projectName);
+ }else{
+ bmlResponse = bmlHelper.upload(userName, inputStream, fileName);
+ }
+ resourceId = bmlResponse.get("resourceId").toString();
+ version = bmlResponse.get("version").toString();
+ } else {
+ // 更新文件
+ Map<String, Object> bmlResponse = bmlHelper.update(userName, resourceId, inputStream);
+ resourceId = bmlResponse.get("resourceId").toString();
+ version = bmlResponse.get("version").toString();
+ }
+ return Message.messageToResponse(Message.ok().data("resourceId", resourceId).data("version", version));
}
- // TODO: 2019/5/28 close 流
- if(inputStream != null) inputStream.close();
- return Message.messageToResponse(Message.ok().data("resourceId",resourceId).data("version",version));
}
}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java
index f434f89..9016049 100644
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java
@@ -18,25 +18,22 @@
import com.webank.wedatasphere.linkis.common.io.FsPath;
+import com.webank.wedatasphere.linkis.common.io.FsWriter;
import com.webank.wedatasphere.linkis.common.io.MetaData;
import com.webank.wedatasphere.linkis.common.io.Record;
import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet;
-import com.webank.wedatasphere.linkis.filesystem.conf.WorkSpaceConfiguration;
import com.webank.wedatasphere.linkis.filesystem.entity.DirFileTree;
+import com.webank.wedatasphere.linkis.filesystem.entity.LogLevel;
import com.webank.wedatasphere.linkis.filesystem.exception.WorkSpaceException;
-import com.webank.wedatasphere.linkis.filesystem.restful.remote.FsRestfulRemote;
+import com.webank.wedatasphere.linkis.filesystem.exception.WorkspaceExceptionManager;
import com.webank.wedatasphere.linkis.filesystem.service.FsService;
-import com.webank.wedatasphere.linkis.filesystem.util.Constants;
-import com.webank.wedatasphere.linkis.filesystem.util.FsUtil;
-import com.webank.wedatasphere.linkis.filesystem.util.FsUtil$;
import com.webank.wedatasphere.linkis.filesystem.util.WorkspaceUtil;
+import com.webank.wedatasphere.linkis.filesystem.validator.PathValidator$;
import com.webank.wedatasphere.linkis.server.Message;
import com.webank.wedatasphere.linkis.server.security.SecurityFilter;
-import com.webank.wedatasphere.linkis.storage.FSFactory;
import com.webank.wedatasphere.linkis.storage.LineMetaData;
import com.webank.wedatasphere.linkis.storage.LineRecord;
import com.webank.wedatasphere.linkis.storage.csv.CSVFsWriter;
-import com.webank.wedatasphere.linkis.storage.domain.Column;
import com.webank.wedatasphere.linkis.storage.domain.FsPathListWithError;
import com.webank.wedatasphere.linkis.storage.excel.ExcelFsWriter;
import com.webank.wedatasphere.linkis.storage.excel.ExcelStorageReader;
@@ -47,9 +44,12 @@
import com.webank.wedatasphere.linkis.storage.resultset.table.TableMetaData;
import com.webank.wedatasphere.linkis.storage.resultset.table.TableRecord;
import com.webank.wedatasphere.linkis.storage.script.*;
+import com.webank.wedatasphere.linkis.storage.source.FileSource;
+import com.webank.wedatasphere.linkis.storage.source.FileSource$;
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.commons.math3.util.Pair;
+import org.apache.http.Consts;
import org.codehaus.jackson.JsonNode;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -72,86 +72,64 @@
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+
+import static com.webank.wedatasphere.linkis.filesystem.conf.WorkSpaceConfiguration.*;
+import static com.webank.wedatasphere.linkis.filesystem.constant.WorkSpaceConstants.*;
/**
- * johnnwang
- * 2018/10/25
+ * johnnwang
+ * 2018/10/25
*/
@Produces(MediaType.APPLICATION_JSON)
@Consumes({MediaType.APPLICATION_JSON, MediaType.MULTIPART_FORM_DATA})
@Component
@Path("filesystem")
-public class FsRestfulApi implements FsRestfulRemote {
+public class FsRestfulApi {
+
@Autowired
private FsService fsService;
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
- private FileSystem getFileSystem(FsPath fsPath, String userName) throws IOException {
- FileSystem fileSystem = (FileSystem) FSFactory.getFsByProxyUser(fsPath, userName);
- fileSystem.init(new HashMap<>());
- return fileSystem;
- }
-
- private void fsValidate(FileSystem fileSystem) throws WorkSpaceException {
- if (fileSystem == null){
-
- throw new WorkSpaceException("The user has obtained the filesystem for more than " + FsUtil.FILESYSTEM_GET_TIMEOUT().getValue().toString()
- + "ms. Please contact the administrator.（用户获取filesystem的时间超过" + FsUtil.FILESYSTEM_GET_TIMEOUT().getValue().toString() + "ms，请联系管理员）");
- }
- }
-
@GET
@Path("/getUserRootPath")
- @Override
- public Response getUserRootPath(@Context HttpServletRequest req,@QueryParam("pathType")String pathType) throws IOException, WorkSpaceException {
+ public Response getUserRootPath(@Context HttpServletRequest req, @QueryParam("pathType") String pathType) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
- String path = null;
- String returnType = null;
- if(pathType.equals("hdfs")){
- if (WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue().toString().endsWith("/")){
- path = WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue() + userName + WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_SUFFIX.getValue();
- }else{
- path = WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue() + "/" + userName + WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_SUFFIX.getValue();
- }
- returnType = "HDFS";
- }else {
- if (WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue().toString().endsWith("/")){
- path = WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue() + userName + "/";
- }else{
- path = WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue() + "/" + userName + "/";
- }
- returnType = "Local";
+ String hdfsUserRootPathPrefix = WorkspaceUtil.suffixTuning(HDFS_USER_ROOT_PATH_PREFIX.getValue());
+ String hdfsUserRootPathSuffix = HDFS_USER_ROOT_PATH_SUFFIX.getValue();
+ String localUserRootPath = WorkspaceUtil.suffixTuning(LOCAL_USER_ROOT_PATH.getValue());
+ String path;
+ String returnType;
+ if (StorageUtils.HDFS().equalsIgnoreCase(pathType)) {
+ path = hdfsUserRootPathPrefix + userName + hdfsUserRootPathSuffix;
+ returnType = StorageUtils.HDFS().toUpperCase();
+ } else {
+ path = localUserRootPath + userName;
+ returnType = LOCAL_RETURN_TYPE;
}
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- if (fileSystem != null &&!fileSystem.exists(fsPath)) {
- throw new WorkSpaceException("User local root directory does not exist, please contact administrator to add（用户本地根目录不存在,请联系管理员添加)");
+ if (!fileSystem.exists(fsPath)) {
+ throw WorkspaceExceptionManager.createException(80003);
}
- if (fileSystem == null) {path = null;}
- return Message.messageToResponse(Message.ok().data("user"+returnType+"RootPath", path));
+ return Message.messageToResponse(Message.ok().data(String.format("user%sRootPath", returnType), path));
}
@POST
@Path("/createNewDir")
- @Override
public Response createNewDir(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
String path = json.get("path").getTextValue();
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("path:(路径：)" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
WorkspaceUtil.fileAndDirNameSpecialCharCheck(path);
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
if (fileSystem.exists(fsPath)) {
- throw new WorkSpaceException("The created folder name is duplicated!(创建的文件夹名重复!)");
+ throw WorkspaceExceptionManager.createException(80005);
}
fileSystem.mkdirs(fsPath);
return Message.messageToResponse(Message.ok());
@@ -159,18 +137,17 @@
@POST
@Path("/createNewFile")
- @Override
public Response createNewFile(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
String path = json.get("path").getTextValue();
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
+ WorkspaceUtil.fileAndDirNameSpecialCharCheck(path);
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
if (fileSystem.exists(fsPath)) {
- throw new WorkSpaceException("The file name created is duplicated!(创建的文件名重复!)");
+ throw WorkspaceExceptionManager.createException(80006);
}
fileSystem.createNewFile(fsPath);
return Message.messageToResponse(Message.ok());
@@ -178,25 +155,28 @@
@POST
@Path("/rename")
- @Override
public Response rename(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException {
String oldDest = json.get("oldDest").getTextValue();
String newDest = json.get("newDest").getTextValue();
String userName = SecurityFilter.getLoginUsername(req);
- if (StringUtils.isEmpty(oldDest)) {
- throw new WorkSpaceException("Path(路径)：" + oldDest + "Is empty!(为空！)");
+ if (FILESYSTEM_PATH_CHECK_TRIGGER.getValue()) {
+ LOGGER.info(String.format("path check trigger is open,now check the path,oldDest:%s,newDest:%s", oldDest, newDest));
+ PathValidator$.MODULE$.validate(oldDest, userName);
+ PathValidator$.MODULE$.validate(newDest, userName);
}
- WorkspaceUtil.fileAndDirNameSpecialCharCheck(newDest);
+ if (StringUtils.isEmpty(oldDest)) {
+ throw WorkspaceExceptionManager.createException(80004, oldDest);
+ }
if (StringUtils.isEmpty(newDest)) {
//No change in file name(文件名字无变化)
return Message.messageToResponse(Message.ok());
}
+ WorkspaceUtil.fileAndDirNameSpecialCharCheck(newDest);
FsPath fsPathOld = new FsPath(oldDest);
FsPath fsPathNew = new FsPath(newDest);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPathOld);
- fsValidate(fileSystem);
if (fileSystem.exists(fsPathNew)) {
- throw new WorkSpaceException("The renamed name is repeated!(重命名的名字重复!)");
+ throw WorkspaceExceptionManager.createException(80007);
}
fileSystem.renameTo(fsPathOld, fsPathNew);
return Message.messageToResponse(Message.ok());
@@ -204,73 +184,60 @@
@POST
@Path("/upload")
- @Override
public Response upload(@Context HttpServletRequest req,
@FormDataParam("path") String path,
FormDataMultiPart form) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
List<FormDataBodyPart> files = form.getFields("file");
for (FormDataBodyPart p : files) {
- InputStream is = p.getValueAs(InputStream.class);
FormDataContentDisposition fileDetail = p.getFormDataContentDisposition();
- String fileName = new String(fileDetail.getFileName().getBytes("ISO8859-1"), "UTF-8");
+ String fileName = new String(fileDetail.getFileName().getBytes(Consts.ISO_8859_1), Consts.UTF_8);
FsPath fsPathNew = new FsPath(fsPath.getPath() + "/" + fileName);
+ WorkspaceUtil.fileAndDirNameSpecialCharCheck(fsPathNew.getPath());
fileSystem.createNewFile(fsPathNew);
- OutputStream outputStream = fileSystem.write(fsPathNew, true);
- IOUtils.copy(is, outputStream);
- StorageUtils.close(outputStream, is, null);
+ try (InputStream is = p.getValueAs(InputStream.class);
+ OutputStream outputStream = fileSystem.write(fsPathNew, true)) {
+ IOUtils.copy(is, outputStream);
+ }
}
return Message.messageToResponse(Message.ok());
}
@POST
@Path("/deleteDirOrFile")
- @Override
public Response deleteDirOrFile(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
String path = json.get("path").getTextValue();
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
if (!fileSystem.exists(fsPath)) {
- throw new WorkSpaceException("The deleted file or folder does not exist!(删除的文件or文件夹不存在!)");
+ throw WorkspaceExceptionManager.createException(80008);
}
if (!fileSystem.canWrite(fsPath.getParent()) || !fileSystem.canExecute(fsPath.getParent())) {
- throw new WorkSpaceException("This user does not have permission to delete this file or folder!(该用户无权删除此文件或文件夹！)");
+ throw WorkspaceExceptionManager.createException(80009);
}
deleteAllFiles(fileSystem, fsPath);
return Message.messageToResponse(Message.ok());
}
- private boolean isInUserWorkspace(String path,String userName){
- String hdfsPath = WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue() + userName + WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_SUFFIX.getValue();
- hdfsPath = hdfsPath.endsWith("/")?hdfsPath.substring(0,hdfsPath.length() -1):hdfsPath;
- String filePath = WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue() + userName;
- return path.startsWith(filePath) || path.startsWith(hdfsPath);
- }
-
@GET
@Path("/getDirFileTrees")
- @Override
public Response getDirFileTrees(@Context HttpServletRequest req,
@QueryParam("path") String path) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
- WorkspaceUtil.pathSafeCheck(path,userName);
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
if (!fileSystem.exists(fsPath)) {
return Message.messageToResponse(Message.ok().data("dirFileTrees", null));
}
@@ -278,7 +245,7 @@
dirFileTree.setPath(fsPath.getSchemaPath());
//if(!isInUserWorkspace(path,userName)) throw new WorkSpaceException("The user does not have permission to view the contents of the directory");
if (!fileSystem.canExecute(fsPath) || !fileSystem.canRead(fsPath)) {
- throw new WorkSpaceException("The user does not have permission to view the contents of the directory(该用户无权限查看该目录的内容)");
+ throw WorkspaceExceptionManager.createException(80010);
}
dirFileTree.setName(new File(path).getName());
dirFileTree.setChildren(new ArrayList<>());
@@ -298,12 +265,67 @@
dirFileTree.getChildren().add(dirFileTreeChildren);
}
}
- Message message = Message.ok();
- /*if (fsPathListWithError != null &&!StringUtils.isEmpty(fsPathListWithError.getError())){
- message.data("msg", fsPathListWithError.getError());
- }*/
- message.data("dirFileTrees", dirFileTree);
- return Message.messageToResponse(message);
+ return Message.messageToResponse(Message.ok().data("dirFileTrees", dirFileTree));
+ }
+
+ @POST
+ @Path("/download")
+ public void download(@Context HttpServletRequest req,
+ @Context HttpServletResponse response,
+ @RequestBody Map<String, String> json) throws IOException, WorkSpaceException {
+ InputStream inputStream = null;
+ ServletOutputStream outputStream = null;
+ PrintWriter writer = null;
+ try {
+ String charset = json.get("charset");
+ String userName = SecurityFilter.getLoginUsername(req);
+ String path = json.get("path");
+ if (StringUtils.isEmpty(path)) {
+ throw WorkspaceExceptionManager.createException(80004, path);
+ }
+ if (StringUtils.isEmpty(charset)) {
+ charset = Consts.UTF_8.toString();
+ }
+ FsPath fsPath = new FsPath(path);
+ // TODO: 2018/11/29 Judging the directory, the directory cannot be downloaded(判断目录,目录不能下载)
+ FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
+ if (!fileSystem.exists(fsPath)) {
+ throw WorkspaceExceptionManager.createException(8011);
+ }
+ inputStream = fileSystem.read(fsPath);
+ byte[] buffer = new byte[1024];
+ int bytesRead = 0;
+ response.setCharacterEncoding(charset);
+ java.nio.file.Path source = Paths.get(fsPath.getPath());
+ String contentType = Files.probeContentType(source);
+ if (!StringUtils.isEmpty(contentType)) {
+ response.addHeader("Content-Type", contentType);
+ } else {
+ response.addHeader("Content-Type", "multipart/form-data");
+ }
+ String downloadFileName = new File(fsPath.getPath()).getName();
+ WorkspaceUtil.downloadResponseHeadCheck(downloadFileName);
+ response.addHeader("Content-Disposition", "attachment;filename=" + new File(path).getName());
+ outputStream = response.getOutputStream();
+ while ((bytesRead = inputStream.read(buffer, 0, 1024)) != -1) {
+ outputStream.write(buffer, 0, bytesRead);
+ }
+ } catch (Exception e) {
+ LOGGER.error("download error(下载出错)：", e);
+ response.reset();
+ response.setCharacterEncoding(Consts.UTF_8.toString());
+ response.setContentType("text/plain; charset=utf-8");
+ writer = response.getWriter();
+ writer.append("download error(下载出错)");
+ writer.flush();
+ } finally {
+ if (outputStream != null) {
+ outputStream.flush();
+ }
+ IOUtils.closeQuietly(outputStream);
+ IOUtils.closeQuietly(inputStream);
+ IOUtils.closeQuietly(writer);
+ }
}
/**
@@ -313,79 +335,10 @@
* @throws IOException
*/
@POST
- @Path("/download")
- @Override
- public void download(@Context HttpServletRequest req,
- @Context HttpServletResponse response,
- @RequestBody Map<String, String> json) throws IOException, WorkSpaceException {
- FileSystem fileSystem = null;
- InputStream inputStream = null;
- ServletOutputStream outputStream = null;
- try {
- String charset = json.get("charset");
- String userName = SecurityFilter.getLoginUsername(req);
- String path = json.get("path");
- if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
- }
- if (StringUtils.isEmpty(charset)) {
- charset = "utf-8";
- }
- FsPath fsPath = new FsPath(path);
- //// TODO: 2018/11/29 Judging the directory, the directory cannot be downloaded(判断目录,目录不能下载)
- fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
- if (!fileSystem.exists(fsPath)) {
- throw new WorkSpaceException("The downloaded directory does not exist!(下载的目录不存在!)");
- }
- inputStream = fileSystem.read(fsPath);
- byte[] buffer = new byte[1024];
- int bytesRead = 0;
- response.setCharacterEncoding(charset);
- java.nio.file.Path source = Paths.get(fsPath.getPath());
- String contentType = Files.probeContentType(source);
- if(!StringUtils.isEmpty(contentType)) {
- response.addHeader("Content-Type", contentType);
- } else {
- response.addHeader("Content-Type", "multipart/form-data");
- }
- String downloadFileName = new File(fsPath.getPath()).getName();
- WorkspaceUtil.downloadResponseHeadCheck(downloadFileName);
- response.addHeader("Content-Disposition", "attachment;filename="
- + downloadFileName);
- outputStream = response.getOutputStream();
- while ((bytesRead = inputStream.read(buffer, 0, 1024)) != -1) {
- outputStream.write(buffer, 0, bytesRead);
- }
- } catch (Exception e) {
- LOGGER.error("download error(下载出错)：",e);
- response.reset();
- response.setCharacterEncoding("UTF-8");
- response.setContentType("text/plain; charset=utf-8");
- PrintWriter writer = response.getWriter();
- writer.append("download error(下载出错)");
- writer.flush();
- writer.close();
- } finally {
- if (outputStream != null) {
- outputStream.flush();
- }
- StorageUtils.close(outputStream, inputStream, null);
- }
- }
-
- /**
- * @param req
- * @param response
- * @param json
- * @throws IOException
- */
- @POST
@Path("/resultdownload")
- @Override
public void resultDownload(@Context HttpServletRequest req,
- @Context HttpServletResponse response,
- @RequestBody Map<String, String> json) throws IOException, WorkSpaceException {
+ @Context HttpServletResponse response,
+ @RequestBody Map<String, String> json) throws IOException, WorkSpaceException {
FileSystem fileSystem = null;
InputStream inputStream = null;
ServletOutputStream outputStream = null;
@@ -395,17 +348,16 @@
String userName = SecurityFilter.getLoginUsername(req);
String path = json.get("path");
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
if (StringUtils.isEmpty(charset)) {
charset = "utf-8";
}
- LOGGER.info("resultdownload:"+userName+",path:"+path);
+ LOGGER.info("resultdownload:" + userName + ",path:" + path);
FsPath fsPath = new FsPath(path);
fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
if (!fileSystem.exists(fsPath)) {
- throw new WorkSpaceException("The downloaded directory does not exist!(下载的目录不存在!)");
+ throw WorkspaceExceptionManager.createException(8011);
}
ResultSetFactory instance = ResultSetFactory$.MODULE$.getInstance();
ResultSet<? extends MetaData, ? extends Record> resultSet = instance.getResultSetByPath(fsPath);
@@ -426,17 +378,17 @@
//字段之间tab分割
String rs = org.apache.commons.lang.StringUtils.join(resulstsetColumn, "\t");
outputStream.write(rs.getBytes("utf-8"));
- outputStream.write(System.getProperty("line.separator").getBytes());
+ outputStream.write(System.getProperty("line.separator").getBytes());
resulstsetColumn.clear();
- }
+ }
}
if (metaData instanceof LineMetaData) {
- while (resultSetReader.hasNext()) {
+ while (resultSetReader.hasNext()) {
Record record = resultSetReader.getRecord();
LineRecord lineRecord = (LineRecord) record;
- outputStream.write(lineRecord.getLine().getBytes("utf-8"));
- outputStream.write(System.getProperty("line.separator").getBytes());
- }
+ outputStream.write(lineRecord.getLine().getBytes("utf-8"));
+ outputStream.write(System.getProperty("line.separator").getBytes());
+ }
}
} catch (Exception e) {
LOGGER.error("resultset download error(结果集下载出错)：", e);
@@ -448,190 +400,60 @@
writer.flush();
writer.close();
} finally {
- resultSetReader.close();
+ resultSetReader.close();
if (outputStream != null) {
outputStream.flush();
}
StorageUtils.close(outputStream, inputStream, null);
}
}
-
@GET
@Path("/isExist")
- @Override
public Response isExist(@Context HttpServletRequest req,
@QueryParam("path") String path) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
FsPath fsPath = new FsPath(path);
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
return Message.messageToResponse(Message.ok().data("isExist", fileSystem.exists(fsPath)));
}
- /**
- * @param req
- * @param path
- * @param page
- * @param pageSize
- * @param charset
- * @return
- * @throws IOException
- */
@GET
@Path("/openFile")
- @Override
public Response openFile(@Context HttpServletRequest req,
@QueryParam("path") String path,
- @QueryParam("page") Integer page,
- @QueryParam("pageSize") Integer pageSize,
- @QueryParam("charset") String charset) throws IOException, WorkSpaceException {
+ @DefaultValue("1") @QueryParam("page") Integer page,
+ @DefaultValue("5000") @QueryParam("pageSize") Integer pageSize,
+ @DefaultValue("utf-8") @QueryParam("charset") String charset) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
Message message = Message.ok();
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "Is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
- if (StringUtils.isEmpty(page)) {
- page = 1;
- }
//Throws an exception if the file does not have read access(如果文件没读权限，抛出异常)
if (!fileSystem.canRead(fsPath)) {
- throw new WorkSpaceException("This user has no permission to read this file!");
+ throw WorkspaceExceptionManager.createException(80012);
}
- Object fileContent = null;
- Integer totalLine = null;
- Integer totalPage = 0;
- String type = WorkspaceUtil.getOpenFileTypeByFileName(path);
- if ("script".equals(type)) {
- ScriptFsReader scriptFsReader = ScriptFsReader.getScriptFsReader(fsPath, StringUtils.isEmpty(charset) ? "utf-8" : charset, fileSystem.read(fsPath));
- MetaData metaData = scriptFsReader.getMetaData();
- ScriptMetaData scriptMetaData = (ScriptMetaData) metaData;
- Variable[] variables = scriptMetaData.getMetaData();
- StringBuilder stringBuilder = new StringBuilder();
- List<String> recordList = new ArrayList<>();
- Map<String, Object> params = VariableParser.getMap(variables);
- while (scriptFsReader.hasNext()) {
- ScriptRecord scriptRecord = (ScriptRecord) scriptFsReader.getRecord();
- recordList.add(scriptRecord.getLine());
+ FileSource fileSource = null;
+ try {
+ fileSource = FileSource$.MODULE$.create(fsPath, fileSystem);
+ if (FileSource$.MODULE$.isResultSet(fsPath.getPath())) {
+ fileSource = fileSource.page(page, pageSize);
}
- if (pageSize == null || pageSize >= recordList.size()) {
- page = 1;
- totalPage = 1;
- for (int i = 0; i < recordList.size(); i++) {
- String r = recordList.get(i);
- if ("".equals(r)) {
- stringBuilder.append("\n");
- } else {
- if (i != recordList.size() - 1) {
- stringBuilder.append(r + "\n");
- } else {
- stringBuilder.append(r);
- }
- }
- }
- } else {
- if (recordList.size() % pageSize == 0) {
- totalPage = recordList.size() / pageSize;
- } else {
- totalPage = recordList.size() / pageSize + 1;
- }
- if (page >= totalPage) {
- page = totalPage;
- for (int i = (page - 1) * pageSize; i <= recordList.size() - 1; i++) {
- String r = recordList.get(i);
- if ("".equals(r)) {
- stringBuilder.append("\n");
- } else {
- if (i != recordList.size() - 1) {
- stringBuilder.append(r + "\n");
- } else {
- stringBuilder.append(r);
- }
- }
- }
- } else {
- for (int i = (page - 1) * pageSize; i <= page * pageSize - 1; i++) {
- String r = recordList.get(i);
- if ("".equals(r)) {
- stringBuilder.append("\n");
- } else {
- if (i != page * pageSize - 1) {
- stringBuilder.append(r + "\n");
- } else {
- stringBuilder.append(r);
- }
- }
- }
- }
- }
- totalLine = recordList.size();
- fileContent = stringBuilder.toString();
- scriptFsReader.close();
- message.data("params", params);
- message.data("type", "script/text");
- } else if ("resultset".equals(type)) {
- //返回metadata
- ResultSetFactory instance = ResultSetFactory$.MODULE$.getInstance();
- ResultSet<? extends MetaData, ? extends Record> resultSet = instance.getResultSetByPath(fsPath);
- com.webank.wedatasphere.linkis.common.io.resultset.ResultSetReader<? extends MetaData, ? extends Record> resultSetReader = ResultSetReader.getResultSetReader(resultSet, fileSystem.read(fsPath));
- MetaData metaData = resultSetReader.getMetaData();
- if (StringUtils.isEmpty(pageSize)) {
- pageSize = 5000;
- }
- page = 1;
- Integer rowNum = 0;
- if (metaData instanceof LineMetaData) {
- LineMetaData lineMetaData = (LineMetaData) metaData;
- message.data("metadata", lineMetaData.getMetaData());
- StringBuilder stringBuilder = new StringBuilder();
- for (int i = 0; i < pageSize; i++) {
- if (resultSetReader.hasNext()) {
- rowNum++;
- Record record = resultSetReader.getRecord();
- LineRecord lineRecord = (LineRecord) record;
- stringBuilder.append(lineRecord.getLine() + "\n");
- } else {
- break;
- }
- }
- fileContent = stringBuilder.toString();
- } else if (metaData instanceof TableMetaData) {
- List<String> resultsetMetaDataList = new ArrayList<>();
- TableMetaData tableMetaData = (TableMetaData) metaData;
- Column[] columns = tableMetaData.columns();
- for (Column column : columns) {
- resultsetMetaDataList.add(column.toString());
- }
- message.data("metadata", resultsetMetaDataList);
- List<ArrayList<String>> resultsetRow = new ArrayList<>();
- for (int i = 0; i < pageSize; i++) {
- if (resultSetReader.hasNext()) {
- rowNum++;
- Record record = resultSetReader.getRecord();
- TableRecord tableRecord = (TableRecord) record;
- ArrayList<String> resulstsetColumn = new ArrayList<>();
- Object[] row = tableRecord.row();
- for (Object o : row) {
- resulstsetColumn.add(o == null ? "NULL" : o.toString());
- }
- resultsetRow.add(resulstsetColumn);
- } else {
- break;
- }
- }
- fileContent = resultsetRow;
- }
- totalLine = rowNum;
- resultSetReader.close();
- message.data("type", resultSet.resultSetType());
+ Pair<Object, ArrayList<String[]>> result = fileSource.collect();
+ IOUtils.closeQuietly(fileSource);
+ message.data("metadata", result.getFirst()).data("fileContent", result.getSecond());
+ message.data("type", fileSource.getParams().get("type"));
+ message.data("totalLine", Integer.valueOf(fileSource.getParams().get("totalLine")));
+ return Message.messageToResponse(message.data("page", page).data("totalPage", 0));
+ } finally {
+ IOUtils.closeQuietly(fileSource);
}
- return Message.messageToResponse(message.data("page", page).data("totalLine", totalLine).data("totalPage", totalPage).data("fileContent", fileContent));
}
@@ -643,16 +465,15 @@
*/
@POST
@Path("/saveScript")
- @Override
public Response saveScript(@Context HttpServletRequest req, @RequestBody Map<String, Object> json) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
String path = (String) json.get("path");
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
String charset = (String) json.get("charset");
if (StringUtils.isEmpty(charset)) {
- charset = "utf-8";
+ charset = Consts.UTF_8.toString();
}
String scriptContent = (String) json.get("scriptContent");
Object params = json.get("params");
@@ -660,286 +481,186 @@
Variable[] v = VariableParser.getVariables(map);
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
if (!fileSystem.exists(fsPath)) {
- throw new WorkSpaceException("file does not exist!(文件不存在！)");
+ throw WorkspaceExceptionManager.createException(80013);
}
if (!fileSystem.canWrite(fsPath)) {
- throw new WorkSpaceException("The user has no permission to modify the contents of this file and cannot save it!(该用户无权限对此文件内容进行修改，无法保存！)");
+ throw WorkspaceExceptionManager.createException(80014);
}
- ScriptFsWriter scriptFsWriter = ScriptFsWriter.getScriptFsWriter(fsPath, charset, fileSystem.write(fsPath, true));
- scriptFsWriter.addMetaData(new ScriptMetaData(v));
- String[] split = scriptContent.split("\\n");
- for (int i = 0; i < split.length; i++) {
- if ("".equals(split[i])) {
- split[i] = "\n";
- } else {
- if (i != split.length - 1) {
+ try (
+ ScriptFsWriter scriptFsWriter = ScriptFsWriter.getScriptFsWriter(fsPath, charset, fileSystem.write(fsPath, true));
+ ) {
+ scriptFsWriter.addMetaData(new ScriptMetaData(v));
+ String[] split = scriptContent.split("\\n");
+ for (int i = 0; i < split.length; i++) {
+ if ("".equals(split[i]) || i != split.length - 1) {
split[i] += "\n";
}
+ scriptFsWriter.addRecord(new ScriptRecord(split[i]));
}
- scriptFsWriter.addRecord(new ScriptRecord(split[i]));
+ return Message.messageToResponse(Message.ok());
}
- scriptFsWriter.close();
- return Message.messageToResponse(Message.ok());
}
@GET
@Path("resultsetToExcel")
- @Override
public void resultsetToExcel(
@Context HttpServletRequest req,
@Context HttpServletResponse response,
@QueryParam("path") String path,
- @QueryParam("charset") String charset,
- @QueryParam("outputFileType") String outputFileType,
- @QueryParam("outputFileName") String outputFileName) throws WorkSpaceException, IOException {
- InputStream inputStream = null;
- ByteArrayOutputStream os = null;
+ @DefaultValue("utf-8") @QueryParam("charset") String charset,
+ @DefaultValue("csv") @QueryParam("outputFileType") String outputFileType,
+ @DefaultValue(",") @QueryParam("csvSeperator") String csvSeperator,
+ @DefaultValue("downloadResultset") @QueryParam("outputFileName") String outputFileName,
+ @DefaultValue("result") @QueryParam("sheetName") String sheetName,
+ @DefaultValue("NULL") @QueryParam("nullValue") String nullValue) throws WorkSpaceException, IOException {
ServletOutputStream outputStream = null;
- FileSystem fileSystem = null;
- CSVFsWriter csvfsWriter = null;
- ExcelFsWriter excelFsWriter = null;
- Integer index = 0;
- boolean isLimitDownloadSize = WorkSpaceConfiguration.RESULT_SET_DOWNLOAD_IS_LIMIT.getValue();
- com.webank.wedatasphere.linkis.common.io.resultset.ResultSetReader<? extends MetaData, ? extends Record> resultSetReader = null;
+ FsWriter fsWriter = null;
+ PrintWriter writer = null;
+ FileSource fileSource = null;
try {
- if (StringUtils.isEmpty(charset)) {
- charset = "utf-8";
- }
String userName = SecurityFilter.getLoginUsername(req);
- if (StringUtils.isEmpty(outputFileType)) {
- outputFileType = "csv";
- }
- if (StringUtils.isEmpty(outputFileName)) {
- outputFileName = "result";
- }
- if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "is empty(为空)！");
- }
- String type = WorkspaceUtil.getOpenFileTypeByFileName(path);
- if (!"resultset".equals(type)) {
- throw new WorkSpaceException("unsupported type");
- }
FsPath fsPath = new FsPath(path);
- fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
- response.setCharacterEncoding(charset);
- ResultSetFactory instance = ResultSetFactory$.MODULE$.getInstance();
- ResultSet<? extends MetaData, ? extends Record> resultSet = instance.getResultSetByPath(fsPath);
- resultSetReader = ResultSetReader.getResultSetReader(resultSet, fileSystem.read(fsPath));
- MetaData metaData = resultSetReader.getMetaData();
- if ("csv".equals(outputFileType)) {
- response.addHeader("Content-Type", "text/plain");
- csvfsWriter = CSVFsWriter.getCSVFSWriter(charset, Constants.CSVDEFAULTSEPARATOR);
- if (metaData instanceof TableMetaData) {
- TableMetaData tableMetaData = (TableMetaData) metaData;
- csvfsWriter.addMetaData(tableMetaData);
- while (resultSetReader.hasNext() && (!isLimitDownloadSize || index < WorkSpaceConfiguration.RESULT_SET_DOWNLOAD_MAX_SIZE_CSV.getValue())) {
- index +=1;
- csvfsWriter.addRecord(resultSetReader.getRecord());
- }
- inputStream = csvfsWriter.getCSVStream();
- } else {
- StringBuilder stringBuilder = new StringBuilder();
- LineMetaData lineMetaData = (LineMetaData) metaData;
- if ("NULL".equals(lineMetaData)) {
- stringBuilder.append(lineMetaData.getMetaData());
- stringBuilder.append("\n");
- }
- while (resultSetReader.hasNext() && (!isLimitDownloadSize || index < WorkSpaceConfiguration.RESULT_SET_DOWNLOAD_MAX_SIZE_CSV.getValue())) {
- index +=1;
- LineRecord lineRecord = (LineRecord) resultSetReader.getRecord();
- stringBuilder.append(lineRecord.getLine());
- stringBuilder.append("\n");
- }
- inputStream = new ByteArrayInputStream(stringBuilder.toString().getBytes());
- }
- } else if ("xlsx".equals(outputFileType)) {
- if (!(metaData instanceof TableMetaData)) {
- throw new WorkSpaceException("Only the result set of the table type can be converted to excel!(只有table类型的结果集能转为excel!)");
- }
- TableMetaData tableMetaData = (TableMetaData) metaData;
- response.addHeader("Content-Type", Constants.XLSXRESPONSE);
- excelFsWriter = ExcelFsWriter.getExcelFsWriter(Constants.FILEDEFAULTCHARSET, Constants.DEFAULTSHEETNAME, Constants.DEFAULTDATETYPE);
- excelFsWriter.addMetaData(tableMetaData);
- while (resultSetReader.hasNext() && (!isLimitDownloadSize || index < WorkSpaceConfiguration.RESULT_SET_DOWNLOAD_MAX_SIZE_EXCEL.getValue())) {
- index +=1;
- excelFsWriter.addRecord(resultSetReader.getRecord());
- }
- Workbook workBook = excelFsWriter.getWorkBook();
- os = new ByteArrayOutputStream();
- workBook.write(os);
- byte[] content = os.toByteArray();
- inputStream = new ByteArrayInputStream(content);
- } else {
- throw new WorkSpaceException("unsupported type");
+ FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
+ boolean isLimitDownloadSize = RESULT_SET_DOWNLOAD_IS_LIMIT.getValue();
+ Integer csvDownloadSize = RESULT_SET_DOWNLOAD_MAX_SIZE_CSV.getValue();
+ Integer excelDownloadSize = RESULT_SET_DOWNLOAD_MAX_SIZE_EXCEL.getValue();
+ if (StringUtils.isEmpty(path)) {
+ throw WorkspaceExceptionManager.createException(80004, path);
}
String downloadFileName = new String(outputFileName.getBytes("UTF-8"), "ISO8859-1") + "." + outputFileType;
WorkspaceUtil.downloadResponseHeadCheck(downloadFileName);
response.addHeader("Content-Disposition", "attachment;filename="
+ downloadFileName);
-
+ response.setCharacterEncoding(charset);
outputStream = response.getOutputStream();
- byte[] buffer = new byte[1024];
- int bytesRead = 0;
- while ((bytesRead = inputStream.read(buffer, 0, 1024)) != -1) {
- outputStream.write(buffer, 0, bytesRead);
+ // 前台传""会自动转为null
+ if (nullValue != null && BLANK.equalsIgnoreCase(nullValue)) nullValue = "";
+ fileSource = FileSource$.MODULE$.create(fsPath, fileSystem).addParams("nullValue", nullValue);
+ switch (outputFileType) {
+ case "csv":
+ if (FileSource$.MODULE$.isTableResultSet(fileSource)) {
+ fsWriter = CSVFsWriter.getCSVFSWriter(charset, csvSeperator, outputStream);
+ } else {
+ fsWriter = ScriptFsWriter.getScriptFsWriter(new FsPath(outputFileType), charset, outputStream);
+ }
+ response.addHeader("Content-Type", "text/plain");
+ if (isLimitDownloadSize) {
+ fileSource = fileSource.page(1, csvDownloadSize);
+ }
+ break;
+ case "xlsx":
+ if (!FileSource$.MODULE$.isTableResultSet(fileSource)) {
+ throw WorkspaceExceptionManager.createException(80024);
+ }
+ fsWriter = ExcelFsWriter.getExcelFsWriter(charset, sheetName, DEFAULT_DATE_TYPE, outputStream);
+ response.addHeader("Content-Type", XLSX_RESPONSE_CONTENT_TYPE);
+ if (isLimitDownloadSize) {
+ fileSource = fileSource.page(1, excelDownloadSize);
+ }
+ break;
+ default:
+ WorkspaceExceptionManager.createException(80015);
}
+ fileSource.write(fsWriter);
+ fsWriter.flush();
} catch (Exception e) {
LOGGER.error("resultset to excel/csv error(结果集导出出错)：", e);
response.reset();
- response.setCharacterEncoding("UTF-8");
+ response.setCharacterEncoding(Consts.UTF_8.toString());
response.setContentType("text/plain; charset=utf-8");
- PrintWriter writer = response.getWriter();
+ writer = response.getWriter();
writer.append("resultset to excel/csv error(结果集导出出错)");
writer.flush();
- writer.close();
} finally {
if (outputStream != null) {
outputStream.flush();
}
- StorageUtils.close(outputStream, inputStream, null);
- StorageUtils.close(os);
- if (csvfsWriter != null) {
- csvfsWriter.close();
- }
- if (excelFsWriter != null) {
- excelFsWriter.close();
- }
- if (resultSetReader != null) {
- resultSetReader.close();
- }
+ IOUtils.closeQuietly(fsWriter);
+ IOUtils.closeQuietly(fileSource);
+ IOUtils.closeQuietly(writer);
}
}
@GET
@Path("formate")
- @Override
public Response formate(@Context HttpServletRequest req,
@QueryParam("path") String path,
- @QueryParam("encoding") String encoding,
- @QueryParam("fieldDelimiter") String fieldDelimiter,
- @QueryParam("hasHeader") Boolean hasHeader,
- @QueryParam("quote") String quote,
- @QueryParam("escapeQuotes") Boolean escapeQuotes) throws Exception {
+ @DefaultValue("utf-8") @QueryParam("encoding") String encoding,
+ @DefaultValue(",") @QueryParam("fieldDelimiter") String fieldDelimiter,
+ @DefaultValue("false") @QueryParam("hasHeader") Boolean hasHeader,
+ @DefaultValue("\"") @QueryParam("quote") String quote,
+ @DefaultValue("false") @QueryParam("escapeQuotes") Boolean escapeQuotes) throws Exception {
String userName = SecurityFilter.getLoginUsername(req);
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
String suffix = path.substring(path.lastIndexOf("."));
FsPath fsPath = new FsPath(path);
Map<String, Object> res = new HashMap<String, Object>();
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
- InputStream in = fileSystem.read(fsPath);
- if (".xlsx".equalsIgnoreCase(suffix) || ".xls".equalsIgnoreCase(suffix)) {
- List<List<String>> info;
- info = ExcelStorageReader.getExcelTitle(in, null, hasHeader, suffix);
- res.put("columnName", info.get(1));
- res.put("columnType", info.get(2));
- res.put("sheetName", info.get(0));
- } else {
- if (StringUtils.isEmpty(encoding)) {
- encoding = Constants.FILEDEFAULTCHARSET;
- }
- if (StringUtils.isEmpty(fieldDelimiter)) {
- fieldDelimiter = Constants.CSVDEFAULTSEPARATOR;
- }
- if (StringUtils.isEmpty(quote)) {
- quote = "\"";
- }
- if (hasHeader == null) {
- hasHeader = false;
- }
- if (escapeQuotes == null) {
- escapeQuotes = false;
- }
- String[][] column = null;
- BufferedReader reader = new BufferedReader(new InputStreamReader(in, encoding));
- String header = reader.readLine();
- if (StringUtils.isEmpty(header)) {
- throw new WorkSpaceException("The file content is empty and cannot be imported!(文件内容为空，不能进行导入操作！)");
- }
- String[] line = header.split(fieldDelimiter);
- int colNum = line.length;
- column = new String[2][colNum];
- if (hasHeader) {
- for (int i = 0; i < colNum; i++) {
- column[0][i] = line[i];
- if (escapeQuotes) {
- try {
- column[0][i] = column[0][i].substring(1, column[0][i].length() - 1);
- } catch (StringIndexOutOfBoundsException e) {
- throw new WorkSpaceException("The header of the file has no qualifiers. Do not check the first behavior header or set no qualifier!(该文件的表头没有限定符，请勿勾选首行为表头或者设置无限定符！)");
- }
- }
- column[1][i] = "string";
- }
+ try (InputStream in = fileSystem.read(fsPath)) {
+ if (".xlsx".equalsIgnoreCase(suffix) || ".xls".equalsIgnoreCase(suffix)) {
+ List<List<String>> info;
+ info = ExcelStorageReader.getExcelTitle(in, null, hasHeader, suffix);
+ res.put("columnName", info.get(1));
+ res.put("columnType", info.get(2));
+ res.put("sheetName", info.get(0));
} else {
- for (int i = 0; i < colNum; i++) {
- column[0][i] = "col_" + (i + 1);
- column[1][i] = "string";
+ String[][] column = null;
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in, encoding));
+ String header = reader.readLine();
+ if (StringUtils.isEmpty(header)) {
+ throw WorkspaceExceptionManager.createException(80016);
}
+ String[] line = header.split(fieldDelimiter, -1);
+ int colNum = line.length;
+ column = new String[2][colNum];
+ if (hasHeader) {
+ for (int i = 0; i < colNum; i++) {
+ column[0][i] = line[i];
+ if (escapeQuotes) {
+ try {
+ column[0][i] = column[0][i].substring(1, column[0][i].length() - 1);
+ } catch (StringIndexOutOfBoundsException e) {
+ throw WorkspaceExceptionManager.createException(80017);
+ }
+ }
+ column[1][i] = "string";
+ }
+ } else {
+ for (int i = 0; i < colNum; i++) {
+ column[0][i] = "col_" + (i + 1);
+ column[1][i] = "string";
+ }
+ }
+ res.put("columnName", column[0]);
+ res.put("columnType", column[1]);
}
- res.put("columnName", column[0]);
- res.put("columnType", column[1]);
+ return Message.messageToResponse(Message.ok().data("formate", res));
}
- StorageUtils.close(null, in, null);
- return Message.messageToResponse(Message.ok().data("formate", res));
}
@GET
@Path("/openLog")
- @Override
public Response openLog(@Context HttpServletRequest req, @QueryParam("path") String path) throws IOException, WorkSpaceException {
String userName = SecurityFilter.getLoginUsername(req);
if (StringUtils.isEmpty(path)) {
- throw new WorkSpaceException("Path(路径)：" + path + "is empty!(为空！)");
+ throw WorkspaceExceptionManager.createException(80004, path);
}
FsPath fsPath = new FsPath(path);
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
- fsValidate(fileSystem);
if (!fileSystem.canRead(fsPath)) {
- throw new WorkSpaceException("This user has no permission to read this log!(该用户无权限读取此日志！)");
+ throw WorkspaceExceptionManager.createException(80018);
}
- String type = WorkspaceUtil.getOpenFileTypeByFileName(path);
- if (!"script".equals(type)) {
- throw new WorkSpaceException("This file is not a log file!(该文件不是日志文件！)");
+ try (FileSource fileSource = FileSource$.MODULE$.create(fsPath, fileSystem).addParams("ifMerge", "false")) {
+ Pair<Object, ArrayList<String[]>> collect = fileSource.collect();
+ StringBuilder[] log = Arrays.stream(new StringBuilder[4]).map(f -> new StringBuilder()).toArray(StringBuilder[]::new);
+ ArrayList<String[]> snd = collect.getSecond();
+ LogLevel start = new LogLevel(LogLevel.Type.ALL);
+ snd.stream().map(f -> f[0]).forEach(s -> WorkspaceUtil.logMatch(s, start).forEach(i -> log[i].append(s).append("\n")));
+ return Message.messageToResponse(Message.ok().data("log", Arrays.stream(log).map(StringBuilder::toString).toArray(String[]::new)));
}
- ScriptFsReader scriptFsReader = ScriptFsReader.getScriptFsReader(fsPath, "utf-8", fileSystem.read(fsPath));
- MetaData metaData = scriptFsReader.getMetaData();
- StringBuilder info = new StringBuilder();
- StringBuilder warn = new StringBuilder();
- StringBuilder error = new StringBuilder();
- StringBuilder all = new StringBuilder();
- StringBuilder tmp = null;
- while (scriptFsReader.hasNext()) {
- ScriptRecord scriptRecord = (ScriptRecord) scriptFsReader.getRecord();
- String line = scriptRecord.getLine();
- all.append(line + "\n");
- if (WorkspaceUtil.logMatch(line, WorkspaceUtil.infoReg)) {
- info.append(line + "\n");
- //tmp = info;
- } else if (WorkspaceUtil.logMatch(line, WorkspaceUtil.errorReg)) {
- error.append(line + "\n");
- //tmp = error;
- } else if (WorkspaceUtil.logMatch(line, WorkspaceUtil.warnReg)) {
- warn.append(line + "\n");
- //tmp = warn;
- }/*else {
- if (tmp !=null){
- tmp.append(line + "\n");
- }
- }*/
- }
- scriptFsReader.close();
- ArrayList<String> log = new ArrayList<>();
- log.add(error.toString());
- log.add(warn.toString());
- log.add(info.toString());
- log.add(all.toString());
- return Message.messageToResponse(Message.ok().data("log", log));
}
private static void deleteAllFiles(FileSystem fileSystem, FsPath fsPath) throws IOException {
@@ -956,4 +677,4 @@
}
fileSystem.delete(fsPath);
}
-}
+}
\ No newline at end of file
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/remote/FsRestfulRemote.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/remote/FsRestfulRemote.java
deleted file mode 100644
index 6a586c2..0000000
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/remote/FsRestfulRemote.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.restful.remote;
-
-import com.webank.wedatasphere.linkis.filesystem.exception.WorkSpaceException;
-import org.codehaus.jackson.JsonNode;
-import org.glassfish.jersey.media.multipart.FormDataMultiPart;
-import org.glassfish.jersey.media.multipart.FormDataParam;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Created by johnnwang on 2018/10/24.
- */
-public interface FsRestfulRemote {
-
- @GetMapping("/api/filesystem/getUserRootPath")
- Response getUserRootPath(@Context HttpServletRequest req,@QueryParam("pathType")String pathType) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/createNewDir")
- Response createNewDir(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/createNewFile")
- Response createNewFile(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/raname")
- Response rename(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/upload")
- Response upload(@Context HttpServletRequest req , @FormDataParam("path") String path, FormDataMultiPart form) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/deleteDirOrFile")
- Response deleteDirOrFile(@Context HttpServletRequest req, JsonNode json) throws IOException, WorkSpaceException;
-
- @GetMapping("/api/filesystem/getFileSystemTree")
- Response getDirFileTrees(@Context HttpServletRequest req, @QueryParam("path")String path) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/download")
- void download(@Context HttpServletRequest req, @Context HttpServletResponse response, @RequestBody Map<String,String> json) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/resultdownload")
- void resultDownload(@Context HttpServletRequest req, @Context HttpServletResponse response, @RequestBody Map<String,String> json) throws IOException, WorkSpaceException;
-
- @GetMapping("/api/filesystem/isExit")
- Response isExist(@Context HttpServletRequest req, @QueryParam("path")String path) throws IOException, WorkSpaceException;
-
- @GetMapping("/api/filesystem/openFile")
- Response openFile(@Context HttpServletRequest req, @QueryParam("path")String path,@QueryParam("page")Integer page,@QueryParam("pageSize") Integer pageSize,
- @QueryParam("charset")String charset) throws IOException, WorkSpaceException;
-
- @PostMapping("/api/filesystem/saveScript")
- Response saveScript(@Context HttpServletRequest req,@RequestBody Map<String,Object> json) throws IOException, WorkSpaceException;
-
- @GetMapping("/api/filesystem/resultsetToExcel")
- void resultsetToExcel(
- @Context HttpServletRequest req,
- @Context HttpServletResponse response,
- @QueryParam("path")String path,@QueryParam("charset") String charset,
- @QueryParam("outputFileType")String outputFileType,
- @QueryParam("outputFileName")String outputFileName) throws WorkSpaceException, IOException;
- @GetMapping("/api/filesystem/formate")
- Response formate(@Context HttpServletRequest req,
- @QueryParam("path") String path,
- @QueryParam("encoding") String encoding,
- @QueryParam("fieldDelimiter") String fieldDelimiter,
- @QueryParam("hasHeader") Boolean hasHeader,
- @QueryParam("quote") String quote,
- @QueryParam("escapeQuotes") Boolean escapeQuotes) throws Exception;
-
- @GetMapping("/api/filesystem/openLog")
- Response openLog(@Context HttpServletRequest req, @QueryParam("path")String path) throws IOException, WorkSpaceException;
-}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/CSVUtil.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/CSVUtil.java
deleted file mode 100644
index f02a5c5..0000000
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/CSVUtil.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.util;
-
-import com.webank.wedatasphere.linkis.storage.domain.Column;
-import com.webank.wedatasphere.linkis.storage.resultset.table.TableRecord;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.List;
-
-/**
- * Created by johnnwang on 2018/11/6.
- */
-@Deprecated
-public class CSVUtil {
-
- public static InputStream createCSV(Column[] columns, List<TableRecord> tableRecords,String seperator){
- StringBuilder stringBuilder = new StringBuilder();
- for (int i = 0; i < columns.length; i++) {
- if (i == columns.length-1){
- stringBuilder.append(columns[i].columnName());
- }else{
- stringBuilder.append(columns[i].columnName()+",");
- }
- }
- stringBuilder.append("\n");
- for (int i = 0; i < tableRecords.size(); i++) {
- Object[] row = tableRecords.get(i).row();
- for (int j = 0; j < row.length; j++) {
- if (j == row.length-1){
- stringBuilder.append(row[j]);
- }else{
- stringBuilder.append(row[j]+",");
- }
-
- }
- if (i != tableRecords.size()-1){
- stringBuilder.append("\n");
- }
- }
- return new ByteArrayInputStream(stringBuilder.toString().getBytes());
- }
-}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/Constants.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/Constants.java
deleted file mode 100644
index c4d263e..0000000
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/Constants.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.util;
-
-/**
- * Created by johnnwang on 2018/10/17.
- */
-public class Constants {
- public static final String APPLICATION_NAME = "cloud-filesystem";
- public static final String XLSXRESPONSE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
- public static final String CHARSETRESPONSE = "utf-8";
- public static final String FILEDEFAULTCHARSET = "utf-8";
- public static final String CSVDEFAULTSEPARATOR = ",";
- public static final String DEFAULTSHEETNAME = "result";
- public static final String DEFAULTDATETYPE = "yyyy-MM-dd HH:mm:ss";
-}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/ExcelUtil.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/ExcelUtil.java
deleted file mode 100644
index 041a4d5..0000000
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/ExcelUtil.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.util;
-
-import com.webank.wedatasphere.linkis.storage.domain.Column;
-import com.webank.wedatasphere.linkis.storage.resultset.table.TableRecord;
-import org.apache.poi.hssf.usermodel.*;
-import org.apache.poi.ss.usermodel.Workbook;
-import org.apache.poi.xssf.usermodel.*;
-
-import java.util.Date;
-import java.util.List;
-
-/**
- * Created by johnnwang on 2018/11/5.
- */
-@Deprecated
-public class ExcelUtil {
-
- public static Workbook createWorkBook(String sheetName, Column[] columns, List<TableRecord> tableRecords){
- HSSFWorkbook wb = new HSSFWorkbook();
- HSSFSheet sheet = wb.createSheet("result");
- HSSFRow tableHead = sheet.createRow(0);
- for (int i = 0; i < columns.length; i++) {
- HSSFCell cell = tableHead.createCell(i);
- cell.setCellValue(columns[i].columnName());
- }
- for (int i = 0; i < tableRecords.size(); i++) {
- HSSFRow tableBody = sheet.createRow(i+1);
- Object[] row = tableRecords.get(i).row();
- for (int j = 0; j < columns.length; j++) {
- HSSFCell cell = tableBody.createCell(j);
- HSSFCellStyle style = wb.createCellStyle();
- HSSFDataFormat format = wb.createDataFormat();
- switch (columns[j].dataType().toString()){
- case "tinyint":
- case "short":
- case "int":
- case "long":
- style.setDataFormat(format.getFormat("0"));
- cell.setCellValue(Long.parseLong(row[j].toString()));
- cell.setCellStyle(style);
- break;
- case "float":
- case "double":
- style.setDataFormat(format.getFormat("0.00E+00"));
- cell.setCellValue(Double.parseDouble(row[j].toString()));
- cell.setCellStyle(style);
- break;
- case "date":
- case "TimestampType":
- style.setDataFormat(format.getFormat("YYYY年MM月dd日 HH:mm:ss"));
- cell.setCellValue(new Date(Long.parseLong(row[j].toString())));
- cell.setCellStyle(style);
- break;
- default:
- style.setDataFormat(format.getFormat("@"));
- cell.setCellValue(row[j].toString());
- cell.setCellStyle(style);
- break;
- }
- }
- }
- return wb;
- }
-
-
- public static Workbook create07WorkBook(String sheetName, Column[] columns, List<TableRecord> tableRecords){
- XSSFWorkbook wb = new XSSFWorkbook();
- XSSFSheet sheet = wb.createSheet("result");
- XSSFRow tableHead = sheet.createRow(0);
- for (int i = 0; i < columns.length; i++) {
- XSSFCell cell = tableHead.createCell(i);
- cell.setCellValue(columns[i].columnName());
- }
- for (int i = 0; i < tableRecords.size(); i++) {
- XSSFRow tableBody = sheet.createRow(i+1);
- Object[] row = tableRecords.get(i).row();
- for (int j = 0; j < columns.length; j++) {
- XSSFCell cell = tableBody.createCell(j);
- XSSFCellStyle style = wb.createCellStyle();
- XSSFDataFormat format = wb.createDataFormat();
- String a = columns[j].dataType().toString();
- switch (columns[j].dataType().toString()){
- case "tinyint":
- case "short":
- case "int":
- case "long":
- style.setDataFormat(format.getFormat("#,#0"));
- cell.setCellValue(Long.parseLong(row[j].toString()));
- cell.setCellStyle(style);
- break;
- case "float":
- case "double":
- style.setDataFormat(format.getFormat("0.00E+00"));
- cell.setCellValue(Double.parseDouble(row[j].toString()));
- cell.setCellStyle(style);
- break;
- case "date":
- case "TimestampType":
- style.setDataFormat(format.getFormat("YYYY年MM月dd日 HH:mm:ss"));
- cell.setCellValue(new Date(Long.parseLong(row[j].toString())));
- cell.setCellStyle(style);
- break;
- default:
- style.setDataFormat(format.getFormat("@"));
- cell.setCellValue(row[j].toString());
- cell.setCellStyle(style);
- break;
- }
- }
- }
- return wb;
- }
-}
diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/WorkspaceUtil.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/WorkspaceUtil.java
index bc55c2c..77bd743 100644
--- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/WorkspaceUtil.java
+++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/util/WorkspaceUtil.java
@@ -16,14 +16,15 @@
package com.webank.wedatasphere.linkis.filesystem.util;
-import static com.webank.wedatasphere.linkis.filesystem.conf.WorkSpaceConfiguration.*;
+
+import com.webank.wedatasphere.linkis.filesystem.entity.LogLevel;
import com.webank.wedatasphere.linkis.filesystem.exception.WorkSpaceException;
-import com.webank.wedatasphere.linkis.storage.utils.StorageUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.util.StringUtils;
+import com.webank.wedatasphere.linkis.filesystem.exception.WorkspaceExceptionManager;
import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
import java.util.function.Function;
import java.util.regex.Pattern;
@@ -31,72 +32,46 @@
* Created by johnnwang on 2018/11/5.
*/
public class WorkspaceUtil {
- private static String[] namenodes;
- private static String linuxUserManagerParentPath;
public static String infoReg = "((19|20)[0-9]{2})-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01]) "
- + "([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" +"\\.\\d{3}\\s*INFO(.*)";
+ + "([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" + "\\.\\d{3}\\s*INFO(.*)";
public static String warnReg = "((19|20)[0-9]{2})-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01]) "
- + "([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" +"\\.\\d{3}\\s*WARN(.*)";
+ + "([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" + "\\.\\d{3}\\s*WARN(.*)";
public static String errorReg = "((19|20)[0-9]{2})-(0?[1-9]|1[012])-(0?[1-9]|[12][0-9]|3[01]) "
- + "([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" +"\\.\\d{3}\\s*ERROR(.*)";
+ + "([01]?[0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]" + "\\.\\d{3}\\s*ERROR(.*)";
+ public static String allReg = "(.*?)";
- public static String getOpenFileTypeByFileName(String path) throws WorkSpaceException {
- if (StringUtils.isEmpty(path)) {
+ public static List<LogLevel.Type> logReg = new ArrayList<>();
+
+ public static List<Integer> logMatch(String code, LogLevel logLevel) {
+ if (logReg.isEmpty()) {
+ synchronized (WorkspaceUtil.class) {
+ if (logReg.isEmpty()) {
+ logReg.add(LogLevel.Type.ERROR);
+ logReg.add(LogLevel.Type.WARN);
+ logReg.add(LogLevel.Type.INFO);
+ }
+ }
}
- if (path.endsWith(".sql")
- || path.endsWith(".hql")
- || path.endsWith(".txt")
- || path.endsWith(".python")
- || path.endsWith(".log")
- || path.endsWith(".r")
- || path.endsWith(".out")
- || path.endsWith(".scala")
- || path.endsWith(".py")
- || path.endsWith(".mlsql")
- || path.endsWith(".jdbc")
- || path.endsWith(".sh")
- ) {
- return "script";
- } else if (path.endsWith(".dolphin")) {
- return "resultset";
+ ArrayList<Integer> result = new ArrayList<>();
+ Optional<LogLevel.Type> any = logReg.stream().filter(r -> Pattern.matches(r.getReg(), code)).findAny();
+ if (any.isPresent()) {
+ result.add(any.get().ordinal());
+ result.add(LogLevel.Type.ALL.ordinal());
+ logLevel.setType(any.get());
} else {
- throw new WorkSpaceException("unsupported type!");
+ result.add(LogLevel.Type.ALL.ordinal());
+ if (logLevel.getType() != LogLevel.Type.ALL) {
+ result.add(logLevel.getType().ordinal());
+ }
}
- }
-
- public static Boolean logMatch(String code ,String pattern){
- return Pattern.matches(pattern,code);
- }
-
- private static final Logger LOGGER = LoggerFactory.getLogger(WorkspaceUtil.class);
-
- //TODO update pathSafeCheck rule
- public static void pathSafeCheck(String path,String userName) throws WorkSpaceException {
- if(!FILESYSTEM_PATH_CHECK_TRIGGER.getValue()) return;
- LOGGER.info("start safe check path params..");
- LOGGER.info(path);
- String userLocalRootPath = suffixTuning(LOCAL_USER_ROOT_PATH.getValue().toString()) + userName;
- String userHdfsRootPath = suffixTuning(HDFS_USER_ROOT_PATH_PREFIX.getValue().toString()) + userName
- + HDFS_USER_ROOT_PATH_SUFFIX.getValue().toString();
- LOGGER.info(userLocalRootPath);
- LOGGER.info(userHdfsRootPath);
- userHdfsRootPath = StringUtils.trimTrailingCharacter(userHdfsRootPath,File.separatorChar);
- if(!path.contains(StorageUtils.FILE_SCHEMA()) && !path.contains(StorageUtils.HDFS_SCHEMA())){
- throw new WorkSpaceException("the path should contain schema");
- }
- if(path.contains("../")){
- throw new WorkSpaceException("Relative path not allowed");
- }
- if(!path.contains(userLocalRootPath) && !path.contains(userHdfsRootPath)){
- throw new WorkSpaceException("The path needs to be within the user's own workspace path");
- }
+ return result;
}
public static Function<String, String> suffixTuningFunction = p -> {
if (p.endsWith(File.separator))
return p;
- else
+ else
return p + File.separator;
};
@@ -107,17 +82,21 @@
public static void fileAndDirNameSpecialCharCheck(String path) throws WorkSpaceException {
String name = new File(path).getName();
- LOGGER.info(path);
- String specialRegEx = "[ _`~!@#$%^&*()+=|{}':;',\\[\\].<>/?~！@#￥%……&*（）——+|{}【】‘；：”“’。，、？]|\n|\r|\t";
+ int i = name.lastIndexOf(".");
+ if (i != -1) {
+ name = name.substring(0, i);
+ }
+ //只支持数字,字母大小写,下划线,中文
+ String specialRegEx = "^[\\w\\u4e00-\\u9fa5]{1,200}$";
Pattern specialPattern = Pattern.compile(specialRegEx);
- if(specialPattern.matcher(name).find()){
- throw new WorkSpaceException("the path exist special char");
+ if (!specialPattern.matcher(name).find()) {
+ WorkspaceExceptionManager.createException(80028);
}
}
public static void downloadResponseHeadCheck(String str) throws WorkSpaceException {
- if(str.contains("\n") || str.contains("\r")){
- throw new WorkSpaceException(String.format("illegal str %s",str));
+ if (str.contains("\n") || str.contains("\r")) {
+ WorkspaceExceptionManager.createException(80030, str);
}
}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLHelper.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLHelper.scala
index e2c673d..0b1c63d 100644
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLHelper.scala
+++ b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLHelper.scala
@@ -21,13 +21,14 @@
import com.webank.wedatasphere.linkis.bml.client.{BmlClient, BmlClientFactory}
import com.webank.wedatasphere.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse}
-import com.webank.wedatasphere.linkis.filesystem.exception.WorkSpaceException
+import com.webank.wedatasphere.linkis.filesystem.exception.WorkspaceExceptionManager
import org.springframework.stereotype.Component
import scala.collection.JavaConversions._
+
/**
- * Created by patinousward
- */
+ * Created by patinousward
+ */
@Component
class BMLHelper {
@@ -35,16 +36,26 @@
val inputStream = new ByteArrayInputStream(content.getBytes("utf-8"))
val client: BmlClient = createBMLClient(userName)
val resource: BmlUploadResponse = client.uploadResource(userName, fileName, inputStream)
- if (!resource.isSuccess) throw new WorkSpaceException("上传失败")
+ if (!resource.isSuccess) throw WorkspaceExceptionManager.createException(80021)
val map = new util.HashMap[String, Object]
map += "resourceId" -> resource.resourceId
map += "version" -> resource.version
}
+ def upload(userName: String, inputStream: InputStream, fileName: String, projectName: String): util.Map[String, Object] = {
+ val client: BmlClient = createBMLClient(userName)
+ val resource: BmlUploadResponse = client.uploadResource(userName, fileName, inputStream)
+ if (!resource.isSuccess) throw WorkspaceExceptionManager.createException(80021)
+ val map = new util.HashMap[String, Object]
+ map += "resourceId" -> resource.resourceId
+ map += "version" -> resource.version
+ }
+
+
def upload(userName: String, inputStream: InputStream, fileName: String): util.Map[String, Object] = {
val client: BmlClient = createBMLClient(userName)
val resource: BmlUploadResponse = client.uploadResource(userName, fileName, inputStream)
- if (!resource.isSuccess) throw new WorkSpaceException("上传失败")
+ if (!resource.isSuccess) throw WorkspaceExceptionManager.createException(80021)
val map = new util.HashMap[String, Object]
map += "resourceId" -> resource.resourceId
map += "version" -> resource.version
@@ -53,7 +64,7 @@
def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = {
val client: BmlClient = createBMLClient(userName)
val resource: BmlUpdateResponse = client.updateResource(userName, resourceId, "", inputStream)
- if (!resource.isSuccess) throw new WorkSpaceException("更新失败")
+ if (!resource.isSuccess) throw WorkspaceExceptionManager.createException(80022)
val map = new util.HashMap[String, Object]
map += "resourceId" -> resource.resourceId
map += "version" -> resource.version
@@ -62,8 +73,8 @@
def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = {
val inputStream = new ByteArrayInputStream(content.getBytes("utf-8"))
val client: BmlClient = createBMLClient(userName)
- val resource: BmlUpdateResponse = client.updateResource(userName, resourceId, UUID.randomUUID().toString+".json", inputStream)
- if (!resource.isSuccess) throw new WorkSpaceException("更新失败")
+ val resource: BmlUpdateResponse = client.updateResource(userName, resourceId, UUID.randomUUID().toString + ".json", inputStream)
+ if (!resource.isSuccess) throw WorkspaceExceptionManager.createException(80022)
val map = new util.HashMap[String, Object]
map += "resourceId" -> resource.resourceId
map += "version" -> resource.version
@@ -72,11 +83,12 @@
def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = {
val client: BmlClient = createBMLClient(userName)
var resource: BmlDownloadResponse = null
- if (version == null) resource = client.downloadResource(userName, resourceId,null) else resource = client.downloadResource(userName, resourceId, version)
- if (!resource.isSuccess) throw new WorkSpaceException("下载失败")
+ if (version == null) resource = client.downloadResource(userName, resourceId, null)
+ else resource = client.downloadResource(userName, resourceId, version)
+ if (!resource.isSuccess) throw WorkspaceExceptionManager.createException(80023)
val map = new util.HashMap[String, Object]
map += "path" -> resource.fullFilePath
- map += "stream" ->resource.inputStream
+ map += "stream" -> resource.inputStream
}
private def inputstremToString(inputStream: InputStream): String = scala.io.Source.fromInputStream(inputStream).mkString
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLScriptReader.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLScriptReader.scala
deleted file mode 100644
index 7ef671d..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLScriptReader.scala
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.webank.wedatasphere.linkis.filesystem.bml
-
-import java.io.{BufferedReader, IOException, InputStream, InputStreamReader}
-
-import com.webank.wedatasphere.linkis.common.io.{FsReader, MetaData, Record}
-import com.webank.wedatasphere.linkis.storage.script.{ParserFactory, ScriptMetaData, ScriptRecord, Variable}
-
-import scala.collection.mutable.ArrayBuffer
-/**
- * Created by patinousward
- */
-class BMLScriptReader private(private val inputStream: InputStream, private val fileName: String) extends FsReader {
-
- private var inputStreamReader: InputStreamReader = _
- private var BufferedReader: BufferedReader = _
- private var metadata: ScriptMetaData = _
-
- private var variables: ArrayBuffer[Variable] = new ArrayBuffer[Variable]
- private var lineText: String = _
-
- @scala.throws[IOException]
- override def getRecord: Record = {
- if (metadata == null) throw new IOException("必须先读取metadata")
- val record = new ScriptRecord(lineText)
- lineText = BufferedReader.readLine()
- record
- }
-
- @scala.throws[IOException]
- override def getMetaData: MetaData = {
- if (metadata == null) init()
- val parser = ParserFactory.listParsers().filter(p => p.belongTo(fileName.substring(fileName.lastIndexOf(".")+1)))
- lineText = BufferedReader.readLine()
- while (hasNext && parser.length > 0 && isMetadata(lineText, parser(0).prefix,parser(0).prefixConf)) {
- variables += parser(0).parse(lineText)
- lineText = BufferedReader.readLine()
- }
- metadata = new ScriptMetaData(variables.toArray)
- metadata
- }
-
- @scala.throws[IOException]
- override def skip(recordNum: Int): Int = ???
-
- @scala.throws[IOException]
- override def getPosition: Long = ???
-
- @scala.throws[IOException]
- override def hasNext: Boolean = lineText != null
-
- @scala.throws[IOException]
- override def available: Long = ???
-
- override def close(): Unit = {
- if (BufferedReader != null) BufferedReader.close()
- if (inputStreamReader != null) inputStreamReader.close()
- if (inputStream != null) inputStream.close()
- }
-
- def init(): Unit = {
- inputStreamReader = new InputStreamReader(inputStream)
- BufferedReader = new BufferedReader(inputStreamReader)
- }
-
- def isMetadata(line: String, prefix: String,prefixConf:String): Boolean = {
- val regex = ("\\s*" + prefix + "\\s*(.+)\\s*" + "=" + "\\s*(.+)\\s*").r
- line match {
- case regex(_, _) => true
- case _ => {
- val split: Array[String] = line.split("=")
- if (split.size != 2) return false
- if (split(0).split(" ").filter(_ != "").size != 4) return false
- if (!split(0).split(" ").filter(_ != "")(0).equals(prefixConf)) return false
- true
- }
- }
- }
-}
-
-object BMLScriptReader {
- def getBMLScriptReader(inputStream: InputStream, fileName: String): BMLScriptReader = new BMLScriptReader(inputStream,fileName)
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLScriptWriter.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLScriptWriter.scala
deleted file mode 100644
index 18c2b34..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/bml/BMLScriptWriter.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.webank.wedatasphere.linkis.filesystem.bml
-
-import java.io.{ByteArrayInputStream, IOException, InputStream}
-
-import com.webank.wedatasphere.linkis.common.io.{FsWriter, MetaData, Record}
-import com.webank.wedatasphere.linkis.storage.script.{Compaction, ScriptMetaData, ScriptRecord}
-/**
- * Created by patinousward
- */
-class BMLScriptWriter private (private val fileName: String) extends FsWriter {
-
- private val stringBuilder = new StringBuilder
-
- @scala.throws[IOException]
- override def addMetaData(metaData: MetaData): Unit = {
- val compactions = Compaction.listCompactions().filter(p => p.belongTo(fileName.substring(fileName.lastIndexOf(".")+1)))
- if (compactions.length > 0) metaData.asInstanceOf[ScriptMetaData].getMetaData.map(compactions(0).compact).foreach(f => stringBuilder.append(f + "\n"))
- }
-
- @scala.throws[IOException]
- override def addRecord(record: Record): Unit = {
- val scriptRecord = record.asInstanceOf[ScriptRecord]
- stringBuilder.append(scriptRecord.getLine)
- }
-
- def getInputStream():InputStream= new ByteArrayInputStream(stringBuilder.toString().getBytes("utf-8"))
-
- override def close(): Unit = ???
-
- override def flush(): Unit = ???
-}
-
-object BMLScriptWriter{
- def getBMLScriptWriter(fileName:String) = new BMLScriptWriter(fileName)
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/AbstractPageHelper.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/AbstractPageHelper.scala
deleted file mode 100644
index c1af76c..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/AbstractPageHelper.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.page
-
-import java.util
-
-/**
- * Created by johnnwang on 2019/4/17.
- */
-abstract class AbstractPageHelper(bodyP: util.ArrayList[util.ArrayList[String]], paramsP: util.HashMap[String, String]) extends PageHelper {
-
- def commonInit(total: Int): Unit = {
- var pageNow = params.getOrDefault("page", "1").toInt
- var pageSize = params.getOrDefault("pageSize", "-1").toInt
- if (pageSize > total || pageSize < 0) pageSize = total
- totalPage = if (total % pageSize == 0) total / pageSize else total / pageSize + 1
- //pageNow > total should be no need to judge(pageNow > total应该是不需要判断的)
- if (pageNow < 0) pageNow = 1 else if (pageNow > totalPage) pageNow = totalPage
- fromIndex = pageSize * (pageNow - 1)
- toIndex = if (pageSize * pageNow - 1 > total - 1) total - 1 else pageSize * pageNow - 1
- }
-
- override var totalPage: Int = _
- override protected var body: util.ArrayList[util.ArrayList[String]] = bodyP
- override protected var fromIndex: Int = _
- override protected var toIndex: Int = _
- override protected var params: util.HashMap[String, String] = paramsP
- init
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/OtherPageHelper.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/OtherPageHelper.scala
deleted file mode 100644
index 269096c..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/OtherPageHelper.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.page
-
-import java.util
-import scala.collection.JavaConversions._
-
-/**
- * Created by johnnwang on 2019/4/17.
- */
-class OtherPageHelper(bodyP: util.ArrayList[util.ArrayList[String]], paramsP: util.HashMap[String, String]) extends AbstractPageHelper(bodyP, paramsP) {
-
- def startPage(): String = {
- val stringBuilder = new StringBuilder
- val subList: util.List[String] = body(0).subList(fromIndex, toIndex)
- subList.foreach {
- f => f match {
- case "" => stringBuilder.append("\n")
- case _ => stringBuilder.append(f + "\n")
- }
- }
- stringBuilder.toString()
- }
-
- override def init(): Unit = {
- commonInit(body(0).size())
- }
-
-}
-
-
-
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/PageHelper.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/PageHelper.scala
deleted file mode 100644
index 9f14e69..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/PageHelper.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.page
-
-import java.util
-
-/**
- * Created by johnnwang on 2019/4/17.
- */
-trait PageHelper {
- protected var fromIndex:Int
- protected var toIndex:Int
- protected var body:util.ArrayList[util.ArrayList[String]]
- protected var params: util.HashMap[String, String]
- var totalPage:Int
- def init:Unit
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/TablePageHelper.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/TablePageHelper.scala
deleted file mode 100644
index d9ba774..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/page/TablePageHelper.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.page
-
-import java.util
-
-/**
- * Created by johnnwang on 2019/4/17.
- */
-class TablePageHelper(bodyP: util.ArrayList[util.ArrayList[String]], paramsP: util.HashMap[String, String]) extends AbstractPageHelper(bodyP, paramsP) {
-
- def startPage() = {
- body.subList(fromIndex, toIndex)
- }
-
- override def init(): Unit = {
- commonInit(body.size())
- }
-
-}
-
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/quartz/FSQuartz.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/quartz/FSQuartz.scala
index 0cbf585..46e4c89 100644
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/quartz/FSQuartz.scala
+++ b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/quartz/FSQuartz.scala
@@ -35,7 +35,8 @@
try {
f.fs.close()
}catch {
- case e:Exception =>info(e.getMessage);info("Requesting IO-Engine call close failed! But still have to continue to clean up the expired fs!(请求IO-Engine调用close失败！但还是要继续清理过期fs！)")
+ case e: Exception =>
+ info("Requesting IO-Engine call close failed! But still have to continue to clean up the expired fs!", e)
}
list -= f
}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/FileReader.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/FileReader.scala
deleted file mode 100644
index c49ac8e..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/FileReader.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.reader
-
-import java.util
-
-import com.webank.wedatasphere.linkis.common.io.{FsPath, FsReader, MetaData, Record}
-import com.webank.wedatasphere.linkis.storage.fs.FileSystem
-
-/**
- * Created by johnnwang on 2019/4/16.
- */
-trait FileReader {
- protected var fsReader: FsReader[MetaData, Record]
-
- protected var kind: String
-
- def init(fileSystem: FileSystem, fsPath: FsPath, params: util.HashMap[String, String]): Unit
-
- def getHead: util.HashMap[String, Object]
-
- /**
- * Expansion of this piece needs to be optimized(扩容这块需要优化下)
- *
- * @return
- */
- def getBody: util.ArrayList[util.ArrayList[String]]
-
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/LineRSReader.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/LineRSReader.scala
deleted file mode 100644
index 03a9060..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/LineRSReader.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.reader
-
-import java.util
-
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
-
-/**
- * Created by johnnwang on 2019/4/16.
- */
-@Deprecated
-class LineRSReader extends RSReader {
- override def getHead: util.HashMap[String, Object] = {
- val lineMetaData = fsReader.getMetaData.asInstanceOf[LineMetaData]
- //generateMap("metadata", lineMetaData.getMetaData)
- null
- }
-
- override def getBody: util.ArrayList[util.ArrayList[String]] = {
- val body = new util.ArrayList[util.ArrayList[String]]
- val bodyChild = new util.ArrayList[String]
- while (fsReader.hasNext) bodyChild.add(fsReader.getRecord.asInstanceOf[LineRecord].getLine)
- fsReader.close()
- body.add(bodyChild)
- body
- }
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/RSReader.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/RSReader.scala
deleted file mode 100644
index cd68f91..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/RSReader.scala
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.reader
-
-import java.util
-
-import com.webank.wedatasphere.linkis.common.io.{FsPath, FsReader, MetaData, Record}
-import com.webank.wedatasphere.linkis.storage.fs.FileSystem
-import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord}
-import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
-
-/**
- * Created by johnnwang on 2019/4/16.
- */
-class RSReader extends FileReader {
-
- def this(fileSystem: FileSystem, fsPath: FsPath, params: util.HashMap[String, String]) = {
- this()
- init(fileSystem, fsPath, params)
- }
-
- override protected var fsReader: FsReader[MetaData, Record] = _
-
- private var rsType: InnerParent = _
-
- override def init(fileSystem: FileSystem, fsPath: FsPath, params: util.HashMap[String, String]): Unit = {
- val resultset = ResultSetFactory.getInstance.getResultSetByPath(fsPath)
- kind = resultset.resultSetType()
- fsReader = ResultSetReader.getResultSetReader(resultset, fileSystem.read(fsPath)).asInstanceOf[FsReader[MetaData, Record]]
- rsType = fsReader.getMetaData match {
- case t: TableMetaData => new InnerTableReader(t)
- case l: LineMetaData => new InnerLineReader(l)
- }
- }
-
- override def getHead: util.HashMap[String, Object] = rsType.getHead
-
- override def getBody: util.ArrayList[util.ArrayList[String]] = rsType.getBody
-
- class InnerParent {
- def getHead: util.HashMap[String, Object] = null
-
- def getBody: util.ArrayList[util.ArrayList[String]] = null
- }
-
- class InnerLineReader(private val metaData: MetaData) extends InnerParent {
- override def getHead: util.HashMap[String, Object] = {
- val lineMetaData = metaData.asInstanceOf[LineMetaData]
- import scala.collection.JavaConversions._
- val map = new util.HashMap[String, Object]
- map += "metadata" -> lineMetaData.getMetaData
- map += "type" -> kind
- map
- }
-
- override def getBody: util.ArrayList[util.ArrayList[String]] = {
- val body = new util.ArrayList[util.ArrayList[String]]
- val bodyChild = new util.ArrayList[String]
- while (fsReader.hasNext) bodyChild.add(fsReader.getRecord.asInstanceOf[LineRecord].getLine)
- fsReader.close()
- body.add(bodyChild)
- body
- }
- }
-
- class InnerTableReader(private val metaData: MetaData) extends InnerParent {
- override def getHead: util.HashMap[String, Object] = {
- val columns = metaData.asInstanceOf[TableMetaData].columns
- val resultsetMetaDataList = new util.ArrayList[String]()
- columns.foreach(f => resultsetMetaDataList.add(f.toString()))
- import scala.collection.JavaConversions._
- val map = new util.HashMap[String, Object]
- map += "metadata" -> resultsetMetaDataList
- map += "type" -> kind
- map
- }
-
- /**
- * Grab 5000 rows directly here.(这里直接先抓取5000行)
- *
- * @return
- */
- override def getBody: util.ArrayList[util.ArrayList[String]] = {
- val body = new util.ArrayList[util.ArrayList[String]]
- var row = 0
- while (fsReader.hasNext && row <= 5000) {
- val rows = fsReader.getRecord.asInstanceOf[TableRecord].row
- val bodyChild = new util.ArrayList[String]()
- rows.foreach {
- f => if (f == null) bodyChild.add("NULL") else bodyChild.add(f.toString)
- }
- body.add(bodyChild)
- row += 1
- }
- fsReader.close()
- body
- }
- }
-
- override protected var kind: String = _
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/TableRSReader.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/TableRSReader.scala
deleted file mode 100644
index ca69f62..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/TableRSReader.scala
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.reader
-
-import java.util
-
-import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord}
-
-/**
- * Created by johnnwang on 2019/4/16.
- */
-@Deprecated
-class TableRSReader extends RSReader {
- override def getHead: util.HashMap[String, Object] = {
- val columns = fsReader.getMetaData.asInstanceOf[TableMetaData].columns
- val resultsetMetaDataList = new util.ArrayList[String]()
- columns.foreach(f => resultsetMetaDataList.add(f.toString()))
- //generateMap("metadata", resultsetMetaDataList)
- null
- }
-
- /**
- * Grab 5000 rows directly here.(这里直接先抓取5000行)
- *
- * @return
- */
- override def getBody: util.ArrayList[util.ArrayList[String]] = {
- val body = new util.ArrayList[util.ArrayList[String]]
- var row = 0
- while (fsReader.hasNext && row <= 5000) {
- val rows = fsReader.getRecord.asInstanceOf[TableRecord].row
- val bodyChild = new util.ArrayList[String]()
- rows.foreach{
- f => if(f == null) bodyChild.add("NULL") else bodyChild.add(f.toString)
- }
- body.add(bodyChild)
- row += 1
- }
- fsReader.close()
- body
- }
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/TxtFileReader.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/TxtFileReader.scala
deleted file mode 100644
index 91dd442..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/reader/TxtFileReader.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.reader
-
-import java.util
-
-import com.webank.wedatasphere.linkis.common.io.{FsPath, FsReader, MetaData, Record}
-import com.webank.wedatasphere.linkis.filesystem.util.FsUtil
-import com.webank.wedatasphere.linkis.storage.fs.FileSystem
-import com.webank.wedatasphere.linkis.storage.script.{ScriptFsReader, ScriptMetaData, ScriptRecord}
-
-/**
- * Created by johnnwang on 2019/4/16.
- */
-class TxtFileReader extends FileReader {
- override protected var fsReader: FsReader[MetaData, Record] = _
-
- def this(fileSystem: FileSystem, fsPath: FsPath, params: util.HashMap[String, String]) = {
- this()
- init(fileSystem, fsPath, params)
- }
-
- override def getHead: util.HashMap[String, Object] = {
- val variables = fsReader.getMetaData.asInstanceOf[ScriptMetaData].getMetaData
- import scala.collection.JavaConversions._
- val map = new util.HashMap[String, Object]
- map += "params" -> FsUtil.getParams(variables)
- map += "type" -> kind
- map
- }
-
- override def getBody: util.ArrayList[util.ArrayList[String]] = {
- val body = new util.ArrayList[util.ArrayList[String]]
- val bodyChild = new util.ArrayList[String]()
- while (fsReader.hasNext) bodyChild.add(fsReader.getRecord.asInstanceOf[ScriptRecord].getLine)
- fsReader.close()
- body.add(bodyChild)
- body
- }
-
- override def init(fileSystem: FileSystem, fsPath: FsPath, params: util.HashMap[String, String]): Unit = {
- val charset = params.getOrDefault("charset", "utf-8")
- this.fsReader = ScriptFsReader.getScriptFsReader(fsPath, charset, fileSystem.read(fsPath)).asInstanceOf[FsReader[MetaData, Record]]
- //this.fsReader = ScriptFsReader.getScriptFsReader(fsPath, charset, new FileInputStream(fsPath.getPath)).asInstanceOf[FsReader[MetaData, Record]]
- }
-
- override protected var kind: String = "script/text"
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/service/FsService.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/service/FsService.scala
index d251974..d40f040 100644
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/service/FsService.scala
+++ b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/service/FsService.scala
@@ -19,25 +19,25 @@
import java.util.concurrent.{Callable, FutureTask, TimeUnit}
import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.filesystem.cache.FsCache
+import com.webank.wedatasphere.linkis.filesystem.conf.WorkSpaceConfiguration
import com.webank.wedatasphere.linkis.filesystem.entity.FSInfo
-import com.webank.wedatasphere.linkis.filesystem.exception.WorkSpaceException
-import com.webank.wedatasphere.linkis.filesystem.util.FsUtil
+import com.webank.wedatasphere.linkis.filesystem.exception.WorkspaceExceptionManager
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.fs.FileSystem
-import org.slf4j.LoggerFactory
import org.springframework.stereotype.Service
-import scala.collection.mutable.ArrayBuffer
import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
import scala.concurrent._
/**
* Created by johnnwang on 2019/2/11.
*/
@Service
-class FsService {
- private val LOGGER = LoggerFactory.getLogger(getClass)
+class FsService extends Logging {
+
def getFileSystemCache(user: String, fsPath: FsPath): FileSystem = {
if (FsCache.fsInfo.get(user) != null) {
@@ -69,23 +69,28 @@
}
def getFileSystem(user: String, fsPath: FsPath): FileSystem = {
+ var fs:FileSystem = null
val start = System.currentTimeMillis()
val task: FutureTask[FileSystem] = new FutureTask[FileSystem](new Callable[FileSystem] {
override def call(): FileSystem = {
- getFileSystemCache(user, fsPath)
+ fs = Utils.tryAndError(getFileSystemCache(user, fsPath))
+ fs
}
})
- FsUtil.executorService.execute(task)
+ WorkSpaceConfiguration.executorService.execute(task)
+ val timeout: java.lang.Long = WorkSpaceConfiguration.FILESYSTEM_GET_TIMEOUT.getValue
try {
- task.get(FsUtil.FILESYSTEM_GET_TIMEOUT.getValue, TimeUnit.MILLISECONDS)
+ task.get(timeout, TimeUnit.MILLISECONDS)
} catch {
- case e: InterruptedException => LOGGER.info(e.getMessage); task.cancel(true); null
- case e: ExecutionException => LOGGER.info(e.getMessage); task.cancel(true); null
- case e: TimeoutException => LOGGER.info(e.getMessage); task.cancel(true); null
+ case e: InterruptedException => error("Failed to getFileSystem", e); task.cancel(true); null
+ case e: ExecutionException => error("Failed to getFileSystem", e); task.cancel(true); null
+ case e: TimeoutException => error("Failed to getFileSystem", e); task.cancel(true); null
} finally {
val end = System.currentTimeMillis()
- LOGGER.info(s"${user} gets the ${fsPath.getFsType} type filesystem using a total of ${end - start} milliseconds(${user}获取${fsPath.getFsType}类型的filesystem一共使用了${end - start}毫秒)")
+ info(s"${user} gets the ${fsPath.getFsType} type filesystem using a total of ${end - start} milliseconds(${user}获取${fsPath.getFsType}类型的filesystem一共使用了${end - start}毫秒)")
}
+ if(fs == null) throw WorkspaceExceptionManager.createException(80002,timeout,timeout)
+ fs
}
@@ -97,12 +102,12 @@
} catch {
//If rpc fails to get fs, for example, io-engine restarts or hangs.(如果rpc获取fs失败了 比如io-engine重启或者挂掉)
case e: Exception => {
- LOGGER.info("Requesting IO-Engine to initialize fileSystem failed",e)
+ error("Requesting IO-Engine to initialize fileSystem failed", e)
//todo Clean up the cache(清理缓存 目前先遗留)
/*FsCache.fsInfo.foreach{
case (_,list) =>list synchronized list.filter(f =>true).foreach(f =>list -=f)
}*/
- throw new WorkSpaceException("Requesting IO-Engine to initialize fileSystem failed!(请求IO-Engine初始化fileSystem失败！)");
+ throw WorkspaceExceptionManager.createException(80001)
}
}
}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/util/FsUtil.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/util/FsUtil.scala
deleted file mode 100644
index e49ebcf..0000000
--- a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/util/FsUtil.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.filesystem.util
-
-import java.util
-import java.util.concurrent.{Executors, LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit}
-
-import com.webank.wedatasphere.linkis.common.conf.CommonVars
-import com.webank.wedatasphere.linkis.storage.script.Variable
-
-import scala.collection.mutable.ArrayBuffer
-
-/**
- * Created by johnnwang on 2019/2/25.
- */
-object FsUtil {
-
- val FILESYSTEM_GET_TIMEOUT = CommonVars("wds.linkis.workspace.filesystem.get.timeout",2000)
- val FILESYSTEM_FS_THREAD_NUM = CommonVars("wds.linkis.workspace.filesystem.thread.num",10)
- val FILESYSTEM_FS_THREAD_CACHE = CommonVars("wds.linkis.workspace.filesystem.thread.cache",1000)
- //val executorService = Executors.newFixedThreadPool(FILESYSTEM_FS_THREAD_NUM.getValue)
- val executorService =
- new ThreadPoolExecutor(FILESYSTEM_FS_THREAD_NUM.getValue,
- FILESYSTEM_FS_THREAD_NUM.getValue, 0L,
- TimeUnit.MILLISECONDS, new LinkedBlockingQueue[Runnable](FILESYSTEM_FS_THREAD_CACHE.getValue))
-
- @Deprecated
- def getVariables(variable: java.util.LinkedHashMap[String, Object],configuration:java.util.LinkedHashMap[String, Object]): Array[Variable] = {
- import scala.collection.JavaConversions._
- val variables = new ArrayBuffer[Variable]
- if(variable != null) variable foreach {
- case (k, v) => {
- variables += new Variable("variable", null, k, v.toString)
- }
- }
- if(configuration !=null) configuration foreach {
- case (k, v) => {
- v.asInstanceOf[java.util.HashMap[String, String]] foreach {
- case (ck, cv) => variables += new Variable("configuration", k, ck, cv)
- }
- }
- }
- variables.toArray
- }
-
- @Deprecated
- def getParams(variables :Array[Variable]):java.util.Map[String,Object] = {
- val map = new util.HashMap[String,Object]
- val variableMap = new util.HashMap[String,String]
- val configMap = new util.HashMap[String,Object]
- val startMap = new util.HashMap[String,String]
- val runMap = new util.HashMap[String,String]
- val specialMap = new util.HashMap[String,String]
- variables.filter(_.sortParent.equals("variable")).foreach(f => variableMap.put(f.key,f.value))
- val configuration = variables.filter(_.sortParent.equals("configuration"))
- configuration.filter(_.sort.equals("startup")).foreach(f=>startMap.put(f.key,f.value))
- configuration.filter(_.sort.equals("runtime")).foreach(f=>runMap.put(f.key,f.value))
- configuration.filter(_.sort.equals("special")).foreach(f=>specialMap.put(f.key,f.value))
- configMap.put("startup",startMap)
- configMap.put("runtime",runMap)
- configMap.put("special",specialMap)
- map.put("variable",variableMap)
- map.put("configuration",configMap)
- map
- }
-}
diff --git a/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/validator/PathValidator.scala b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/validator/PathValidator.scala
new file mode 100644
index 0000000..e76f308
--- /dev/null
+++ b/publicService/workspace/src/main/scala/com/webank/wedatasphere/linkis/filesystem/validator/PathValidator.scala
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.filesystem.validator
+
+import java.io.File
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.filesystem.conf.WorkSpaceConfiguration._
+import com.webank.wedatasphere.linkis.filesystem.exception.WorkSpaceException
+import com.webank.wedatasphere.linkis.filesystem.util.WorkspaceUtil
+import com.webank.wedatasphere.linkis.server
+import com.webank.wedatasphere.linkis.server.Message
+import com.webank.wedatasphere.linkis.server.security.SecurityFilter
+import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import javax.servlet.http.HttpServletRequest
+import javax.ws.rs.core.Response
+import org.aspectj.lang.ProceedingJoinPoint
+import org.aspectj.lang.annotation.{Around, Aspect, Pointcut}
+import org.aspectj.lang.reflect.MethodSignature
+import org.codehaus.jackson.JsonNode
+import org.springframework.stereotype.Component
+import org.springframework.util.StringUtils
+
+@Aspect
+@Component
+class PathValidator extends Logging {
+
+ @Pointcut("@annotation(javax.ws.rs.Path) && within(com.webank.wedatasphere.linkis.filesystem.restful.api.*)")
+ def restfulResponseCatch(): Unit = {}
+
+ def getPath(args: Array[Object], paramNames: Array[String]): String = {
+ var path: String = null
+ var index: Int = paramNames.indexOf("path")
+ if (index != -1) {
+ path = args(index).asInstanceOf[String]
+ } else {
+ index = paramNames.indexOf("json")
+ if (index != -1) {
+ args(index) match {
+ case j: JsonNode if j.get("path") != null => path = j.get("path").getTextValue
+ case m: java.util.Map[String, Object] if m.get("path") != null => path = m.get("path").asInstanceOf[String]
+ case _ =>
+ }
+ }
+ }
+ path
+ }
+
+ def getUserName(args: Array[Object], paramNames: Array[String]): String = {
+ var username: String = null
+ paramNames.indexOf("req") match {
+ case -1 =>
+ case index: Int => {
+ val proxyUser = paramNames.indexOf("proxyUser")
+ if (proxyUser == -1 || StringUtils.isEmpty(args(proxyUser))) {
+ username = SecurityFilter.getLoginUsername(args(index).asInstanceOf[HttpServletRequest])
+ } else {
+ //增加proxyuser的判断
+ username = args(proxyUser).toString
+ }
+ }
+ }
+ username
+ }
+
+ def checkPath(path: String, username: String) = {
+ //校验path的逻辑
+ val userLocalRootPath: String = WorkspaceUtil.suffixTuning(LOCAL_USER_ROOT_PATH.getValue) +
+ username
+ var userHdfsRootPath: String = WorkspaceUtil.suffixTuning(HDFS_USER_ROOT_PATH_PREFIX.getValue) +
+ username + HDFS_USER_ROOT_PATH_SUFFIX.getValue
+ if (!(path.contains(StorageUtils.FILE_SCHEMA)) && !(path.contains(StorageUtils.HDFS_SCHEMA))) {
+ throw new WorkSpaceException(80025, "the path should contain schema")
+ }
+ userHdfsRootPath = StringUtils.trimTrailingCharacter(userHdfsRootPath, File.separatorChar)
+ if (path.contains("../")) {
+ throw new WorkSpaceException(80026, "Relative path not allowed")
+ }
+ if (!(path.contains(userLocalRootPath)) && !(path.contains(userHdfsRootPath))) {
+ throw new WorkSpaceException(80027, "The path needs to be within the user's own workspace path")
+ }
+ }
+
+ def validate(args: Array[Object], paramNames: Array[String]) = {
+ //获取path:String,json:JsonNode,json:Map中的path 参数
+ val path: String = getPath(args, paramNames)
+ val username: String = getUserName(args, paramNames)
+ if (!StringUtils.isEmpty(path) && !StringUtils.isEmpty(username)) {
+ logger.info(String.format("user:%s,path:%s", username, path))
+ checkPath(path, username)
+ }
+ }
+
+ @Around("restfulResponseCatch()")
+ def dealResponseRestful(proceedingJoinPoint: ProceedingJoinPoint): Object = {
+ val response: Response = server.catchIt {
+ val signature = proceedingJoinPoint.getSignature.asInstanceOf[MethodSignature]
+ logger.info("enter the path validator,the method is {}", signature.getName)
+ if (FILESYSTEM_PATH_CHECK_TRIGGER.getValue) {
+ logger.info("path check trigger is open,now check the path")
+ validate(proceedingJoinPoint.getArgs, signature.getParameterNames)
+ }
+ Message.ok()
+ }
+ if (response.getStatus != 200) response else proceedingJoinPoint.proceed()
+ }
+}
+
+object PathValidator {
+ val validator = new PathValidator()
+
+ def validate(path: String, username: String): Unit = {
+ validator.checkPath(path, username)
+ }
+}
diff --git a/resourceManager/resourcemanagerclient/pom.xml b/resourceManager/resourcemanagerclient/pom.xml
index 7f9341f..fda5b9c 100644
--- a/resourceManager/resourcemanagerclient/pom.xml
+++ b/resourceManager/resourcemanagerclient/pom.xml
@@ -20,7 +20,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/resourceManager/resourcemanagerclient/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/client/ResourceManagerClient.scala b/resourceManager/resourcemanagerclient/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/client/ResourceManagerClient.scala
index fcc5fa3..b7f9240 100644
--- a/resourceManager/resourcemanagerclient/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/client/ResourceManagerClient.scala
+++ b/resourceManager/resourcemanagerclient/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/client/ResourceManagerClient.scala
@@ -23,7 +23,6 @@
import com.webank.wedatasphere.linkis.resourcemanager.utils.RMConfiguration
import com.webank.wedatasphere.linkis.resourcemanager.{ResourceRequestPolicy => _, _}
import com.webank.wedatasphere.linkis.rpc.Sender
-import com.webank.wedatasphere.linkis.rpc.transform.RPCProduct
import javax.annotation.PostConstruct
import org.springframework.stereotype.Component
diff --git a/resourceManager/resourcemanagercommon/pom.xml b/resourceManager/resourcemanagercommon/pom.xml
index 051cec5..25e0b3f 100644
--- a/resourceManager/resourcemanagercommon/pom.xml
+++ b/resourceManager/resourcemanagercommon/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMRPCFormats.scala b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMRPCFormats.scala
index 3e368c9..552dd4b 100644
--- a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMRPCFormats.scala
+++ b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMRPCFormats.scala
@@ -18,7 +18,7 @@
import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInfoSerializer, ModuleInstanceSerializer, ModuleResourceInfoSerializer}
import com.webank.wedatasphere.linkis.rpc.transform.RPCFormats
-import org.json4s.{DefaultFormats, Formats, Serializer}
+import org.json4s.Serializer
import org.springframework.stereotype.Component
/**
diff --git a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/Resource.scala b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/Resource.scala
index c18f929..1e13cfb 100644
--- a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/Resource.scala
+++ b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/Resource.scala
@@ -21,12 +21,12 @@
import com.webank.wedatasphere.linkis.resourcemanager.exception.RMWarnException
import com.webank.wedatasphere.linkis.resourcemanager.utils.RMByteTimeUtils
import org.apache.commons.lang.StringUtils
-import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
+import org.json4s.jackson.Serialization
+import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
import scala.collection.JavaConversions
-import org.json4s.jackson.Serialization
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManagerParam.scala b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManagerParam.scala
index 96364e1..324bd38 100644
--- a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManagerParam.scala
+++ b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManagerParam.scala
@@ -16,9 +16,8 @@
package com.webank.wedatasphere.linkis.resourcemanager
-import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInfo, ModuleResourceInfo}
-import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResultResource}
import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleResourceInfo
/**
diff --git a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInfo.scala b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInfo.scala
index 71ac146..329711e 100644
--- a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInfo.scala
+++ b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInfo.scala
@@ -16,11 +16,11 @@
package com.webank.wedatasphere.linkis.resourcemanager.domain
-import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
-import org.json4s.JsonAST.JObject
-import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceRequestPolicy, ResourceSerializer}
-import com.webank.wedatasphere.linkis.resourcemanager.ResourceRequestPolicy.ResourceRequestPolicy
import com.webank.wedatasphere.linkis.common.ServiceInstance
+import com.webank.wedatasphere.linkis.resourcemanager.ResourceRequestPolicy.ResourceRequestPolicy
+import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceRequestPolicy, ResourceSerializer}
+import org.json4s.JsonAST.JObject
+import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInstanceRecord.scala b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInstanceRecord.scala
index d896d62..b1479ba 100644
--- a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInstanceRecord.scala
+++ b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/ModuleInstanceRecord.scala
@@ -17,7 +17,7 @@
package com.webank.wedatasphere.linkis.resourcemanager.domain
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
+import org.json4s.CustomSerializer
import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
diff --git a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceInfo.scala b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceInfo.scala
index 4e1568f..3d87f14 100644
--- a/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceInfo.scala
+++ b/resourceManager/resourcemanagercommon/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceInfo.scala
@@ -16,11 +16,11 @@
package com.webank.wedatasphere.linkis.resourcemanager.domain
-import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
-import org.json4s.JsonAST.{JObject, JValue}
+import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
+import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
+import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagerserver/bin/start-resourcemanager.sh b/resourceManager/resourcemanagerserver/bin/start-resourcemanager.sh
index 56cab6d..80cc775 100644
--- a/resourceManager/resourcemanagerserver/bin/start-resourcemanager.sh
+++ b/resourceManager/resourcemanagerserver/bin/start-resourcemanager.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/resourceManager/resourcemanagerserver/pom.xml b/resourceManager/resourcemanagerserver/pom.xml
index a02e0ba..ddb8bf7 100644
--- a/resourceManager/resourcemanagerserver/pom.xml
+++ b/resourceManager/resourcemanagerserver/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
@@ -109,56 +109,7 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.6.2.Final</version>
- </dependency>
- <dependency>
- <groupId>net.databinder.dispatch</groupId>
- <artifactId>dispatch-core_${scala.binary.version}</artifactId>
- <version>${dispatch.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>net.databinder.dispatch</groupId>
- <artifactId>dispatch-json4s-jackson_${scala.binary.version}</artifactId>
- <version>${dispatch.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.json4s</groupId>
- <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
- <version>3.2.11</version>
- <exclusions>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -255,11 +206,9 @@
</exclusions>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.jsoup</groupId>
- <artifactId>jsoup</artifactId>
- <version>${jsoup.version}</version>
- </dependency>
+
+
+
</dependencies>
<build>
<plugins>
diff --git a/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmMetaData.java b/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmMetaData.java
index 5e4ea65..c5e6078 100644
--- a/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmMetaData.java
+++ b/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmMetaData.java
@@ -16,8 +16,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.domain;
-import java.util.Date;
-
/**
* Created by shanhuang on 9/11/18.
*/
diff --git a/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmResourceMetaData.java b/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmResourceMetaData.java
index 35e7f14..4406332 100644
--- a/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmResourceMetaData.java
+++ b/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/resourcemanager/domain/EmResourceMetaData.java
@@ -16,8 +16,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.domain;
-import java.util.Date;
-
/**
* Created by shanhuang on 9/11/18.
*/
diff --git a/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/server/rpc/SenderUtils.java b/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/server/rpc/SenderUtils.java
index ff23043..ae3bb99 100644
--- a/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/server/rpc/SenderUtils.java
+++ b/resourceManager/resourcemanagerserver/src/main/java/com/webank/wedatasphere/linkis/server/rpc/SenderUtils.java
@@ -1,4 +1,4 @@
-/*
+package com.webank.wedatasphere.linkis.server.rpc;/*
package com.webank.wedatasphere.linkis.server.rpc;
import com.webank.wedatasphere.linkis.common.ServiceInstance;
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ModuleResourceManager.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ModuleResourceManager.scala
index 5f40d50..0a00a6c 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ModuleResourceManager.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ModuleResourceManager.scala
@@ -16,9 +16,8 @@
package com.webank.wedatasphere.linkis.resourcemanager
-
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import com.webank.wedatasphere.linkis.resourcemanager.domain.{EmResourceMetaData, ModuleResourceInfo, ModuleResourceRecord}
+import com.webank.wedatasphere.linkis.resourcemanager.domain.{EmResourceMetaData, ModuleResourceInfo}
import com.webank.wedatasphere.linkis.resourcemanager.event.notify.{ModuleRegisterEvent, ModuleUnregisterEvent, NotifyRMEventListener}
/**
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMContext.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMContext.scala
index 9152b32..8c84416 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMContext.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RMContext.scala
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.resourcemanager
import com.webank.wedatasphere.linkis.resourcemanager.service.DefaultRMContext
-import com.webank.wedatasphere.linkis.resourcemanager.service.metadata.{ModuleResourceRecordService, ResourceLockService}
import com.webank.wedatasphere.linkis.scheduler.Scheduler
/**
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RequestResourceService.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RequestResourceService.scala
index 28056dd..353ff5f 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RequestResourceService.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/RequestResourceService.scala
@@ -17,14 +17,12 @@
package com.webank.wedatasphere.linkis.resourcemanager
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging}
+import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.resourcemanager.ResourceRequestPolicy.ResourceRequestPolicy
-import com.webank.wedatasphere.linkis.resourcemanager.exception.{RMErrorException, RMWarnException}
+import com.webank.wedatasphere.linkis.resourcemanager.exception.RMWarnException
import com.webank.wedatasphere.linkis.resourcemanager.service.metadata.{ModuleResourceRecordService, UserMetaData, UserResourceRecordService}
import com.webank.wedatasphere.linkis.resourcemanager.utils.YarnUtil
import org.json4s.DefaultFormats
-import org.json4s.jackson.JsonMethods._
-import org.springframework.beans.factory.annotation.Autowired
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManager.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManager.scala
index 1f26014..248f532 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManager.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/ResourceManager.scala
@@ -20,8 +20,6 @@
import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInfo, ModuleResourceInfo}
import com.webank.wedatasphere.linkis.resourcemanager.service.rm.DefaultResourceManager
-import scala.concurrent.duration.TimeUnit
-
/**
* Created by shanhuang on 9/11/18.
*/
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceRecord.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceRecord.scala
index 8232041..87ecbb4 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceRecord.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/domain/UserResourceRecord.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.domain
-
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.Resource
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala
index b756a32..4b1def1 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala
@@ -19,7 +19,6 @@
import com.webank.wedatasphere.linkis.common.listener.ListenerBus
import com.webank.wedatasphere.linkis.resourcemanager.event.RMEvent
import com.webank.wedatasphere.linkis.resourcemanager.schedule.RMEventExecuteRequest
-import com.webank.wedatasphere.linkis.resourcemanager.exception.RMErrorException
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState._
import com.webank.wedatasphere.linkis.scheduler.executer.{AbstractExecutor, ExecuteRequest, SuccessExecuteResponse}
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ClearEvent.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ClearEvent.scala
index 1d6802b..17df052 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ClearEvent.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ClearEvent.scala
@@ -17,7 +17,7 @@
package com.webank.wedatasphere.linkis.resourcemanager.event.notify
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import com.webank.wedatasphere.linkis.resourcemanager.domain.{UserPreUsedResource, UserResourceInfo, UserUsedResource}
+import com.webank.wedatasphere.linkis.resourcemanager.domain.{UserPreUsedResource, UserUsedResource}
import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ModuleEvent.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ModuleEvent.scala
index 8d902b5..2d0c622 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ModuleEvent.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/ModuleEvent.scala
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.event.notify
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import com.webank.wedatasphere.linkis.resourcemanager.RMContext
import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo
import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState._
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEvent.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEvent.scala
index 8ec1a05..c8bb844 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEvent.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEvent.scala
@@ -17,8 +17,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.event.notify
import com.webank.wedatasphere.linkis.common.ServiceInstance
-import com.webank.wedatasphere.linkis.resourcemanager.RMContext
-import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInfo, ModuleInfoSerializer, UserResourceInfo}
import com.webank.wedatasphere.linkis.resourcemanager.event._
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala
index a00723b..33d226c 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala
@@ -17,10 +17,9 @@
package com.webank.wedatasphere.linkis.resourcemanager.event.notify
import com.webank.wedatasphere.linkis.common.listener.ListenerBus
-import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.resourcemanager.event.RMEvent
import com.webank.wedatasphere.linkis.resourcemanager.schedule.RMEventExecuteRequest
-import com.webank.wedatasphere.linkis.resourcemanager.exception.RMErrorException
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState.ExecutorState
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecutorState => _, _}
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventSerializer.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventSerializer.scala
index 0f93773..b28242a 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventSerializer.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventSerializer.scala
@@ -19,11 +19,9 @@
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.resourcemanager.domain._
import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope
-import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
-import org.json4s.JsonAST.{JField, JObject}
+import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
-
-import scala.collection.mutable.ArrayBuffer
+import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/UserEvent.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/UserEvent.scala
index b4070ad..7eccfe6 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/UserEvent.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/UserEvent.scala
@@ -21,8 +21,6 @@
import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope
import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState._
-import scala.collection.mutable.ArrayBuffer
-
/**
* Created by shanhuang on 9/11/18.
*/
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala
index a382ffc..e961814 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala
@@ -16,13 +16,11 @@
package com.webank.wedatasphere.linkis.resourcemanager.listener
-
import java.util
import java.util.concurrent.ConcurrentHashMap
-import java.util.{Collections, Comparator, List => JList, Map => JMap}
+import java.util.{Comparator, Map => JMap}
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.resourcemanager.RMContext
import com.webank.wedatasphere.linkis.resourcemanager.event.notify.{ModuleRegisterEvent, ModuleUnregisterEvent, NotifyRMEvent, NotifyRMEventListener}
import scala.beans.BeanProperty
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/NotifyRMEventPublisher.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/NotifyRMEventPublisher.scala
index 516ab9d..831c4f6 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/NotifyRMEventPublisher.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/NotifyRMEventPublisher.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.resourcemanager.notify
-import java.util.concurrent.{ConcurrentHashMap, ScheduledThreadPoolExecutor}
+import java.util.concurrent.ScheduledThreadPoolExecutor
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.resourcemanager.event.notify._
@@ -25,7 +25,7 @@
import org.json4s._
import org.json4s.jackson.Serialization.{read, write}
-import collection.JavaConversions._
+import scala.collection.JavaConversions._
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/TopicPublisher.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/TopicPublisher.scala
index bba2c4b..98a8136 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/TopicPublisher.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/TopicPublisher.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.notify
-
/**
* Created by shanhuang on 9/11/18.
*/
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperDistributedQueue.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperDistributedQueue.scala
index d31848a..72f1ef7 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperDistributedQueue.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperDistributedQueue.scala
@@ -22,7 +22,7 @@
import org.apache.zookeeper.ZooDefs.Ids
import org.apache.zookeeper.{CreateMode, KeeperException, ZKUtil, ZooKeeper}
-import collection.JavaConversions._
+import scala.collection.JavaConversions._
import scala.collection.mutable
/**
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperUtils.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperUtils.scala
index cd38dd6..82e36f6 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperUtils.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/notify/ZookeeperUtils.scala
@@ -16,12 +16,8 @@
package com.webank.wedatasphere.linkis.resourcemanager.notify
-import java.util.List
-
import com.webank.wedatasphere.linkis.resourcemanager.utils.RMConfiguration
-import org.apache.commons.lang.StringUtils
-import org.apache.zookeeper.{CreateMode, WatchedEvent, Watcher, ZooKeeper}
-import org.apache.zookeeper.data.ACL
+import org.apache.zookeeper.{WatchedEvent, Watcher, ZooKeeper}
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RMMonitorRest.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RMMonitorRest.scala
index 35ba517..e120d85 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RMMonitorRest.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RMMonitorRest.scala
@@ -23,13 +23,12 @@
import com.webank.wedatasphere.linkis.protocol.config.RequestQueryAppConfigWithGlobal
import com.webank.wedatasphere.linkis.protocol.engine.{RequestEngineStatus, RequestUserEngineKill, ResponseEngineStatus, ResponseUserEngineKill}
import com.webank.wedatasphere.linkis.protocol.utils.ProtocolUtils
-import com.webank.wedatasphere.linkis.resourcemanager.{ResourceSerializer, _}
-import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInstanceSerializer, UserResourceMetaData, UserResourceRecord}
+import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInstanceSerializer, UserResourceMetaData}
import com.webank.wedatasphere.linkis.resourcemanager.service.metadata.{ModuleResourceRecordService, UserConfiguration, UserResourceRecordService}
-import com.webank.wedatasphere.linkis.resourcemanager.utils.{RMConfiguration, YarnAppInfo, YarnUtil}
+import com.webank.wedatasphere.linkis.resourcemanager.utils.{RMConfiguration, YarnUtil}
+import com.webank.wedatasphere.linkis.resourcemanager.{ResourceSerializer, _}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState
-import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState.ExecutorState
import com.webank.wedatasphere.linkis.server.Message
import com.webank.wedatasphere.linkis.server.security.SecurityFilter
import javax.servlet.http.HttpServletRequest
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RequestParams.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RequestParams.scala
index 564b595..a1f0d93 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RequestParams.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/restful/RequestParams.scala
@@ -18,14 +18,13 @@
import java.util
import java.util.Map
-import java.util.concurrent.TimeUnit
+import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.resourcemanager.ResourceRequestPolicy.ResourceRequestPolicy
import com.webank.wedatasphere.linkis.resourcemanager._
import org.json4s.JsonAST.JObject
import org.json4s._
import org.json4s.jackson.JsonMethods._
-import com.webank.wedatasphere.linkis.common.utils.Logging
/**
* Created by shanhuang on 2019/1/11.
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala
index daa15d7..76a38fa 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala
@@ -16,8 +16,8 @@
package com.webank.wedatasphere.linkis.resourcemanager.schedule
-import com.webank.wedatasphere.linkis.scheduler.queue.{ConsumerManager, SchedulerEvent}
-import com.webank.wedatasphere.linkis.scheduler.{AbstractScheduler, Scheduler, SchedulerContext}
+import com.webank.wedatasphere.linkis.scheduler.queue.ConsumerManager
+import com.webank.wedatasphere.linkis.scheduler.{AbstractScheduler, SchedulerContext}
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala
index 67b01c0..19986b7 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.schedule
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.scheduler.SchedulerContext
import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory
/**
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMConsumerListenerImpl.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMConsumerListenerImpl.scala
index ae768ee..99cfd10 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMConsumerListenerImpl.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMConsumerListenerImpl.scala
@@ -15,6 +15,7 @@
*/
package com.webank.wedatasphere.linkis.resourcemanager.schedule
+
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.resourcemanager.event.RMEvent
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/DefaultRMContext.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/DefaultRMContext.scala
index fde7d89..4be91bb 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/DefaultRMContext.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/DefaultRMContext.scala
@@ -17,13 +17,11 @@
package com.webank.wedatasphere.linkis.resourcemanager.service
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.resourcemanager._
import com.webank.wedatasphere.linkis.resourcemanager.event.notify.NotifyRMEventListenerBus
import com.webank.wedatasphere.linkis.resourcemanager.schedule.{EventSchedulerContextImpl, EventSchedulerImpl, RMEventExecutorManager}
-
import com.webank.wedatasphere.linkis.resourcemanager.service.metadata.{ModuleResourceRecordService, ResourceLockService}
-
import com.webank.wedatasphere.linkis.resourcemanager.utils.{RMConfiguration, RMUtils}
-import com.webank.wedatasphere.linkis.resourcemanager.{RequestResourceService, _}
import com.webank.wedatasphere.linkis.scheduler.Scheduler
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/DefaultUserMetaData.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/DefaultUserMetaData.scala
index de98472..43c4f1e 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/DefaultUserMetaData.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/DefaultUserMetaData.scala
@@ -17,15 +17,13 @@
package com.webank.wedatasphere.linkis.resourcemanager.service.metadata
import java.util
-import java.util.Map.Entry
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfigWithGlobal, ResponseQueryConfig}
+import com.webank.wedatasphere.linkis.protocol.config.RequestQueryAppConfigWithGlobal
import com.webank.wedatasphere.linkis.protocol.utils.ProtocolUtils
import com.webank.wedatasphere.linkis.resourcemanager.ResourceRequestPolicy._
import com.webank.wedatasphere.linkis.resourcemanager._
import com.webank.wedatasphere.linkis.resourcemanager.exception.RMWarnException
-import com.webank.wedatasphere.linkis.resourcemanager.utils.RMConfiguration
import com.webank.wedatasphere.linkis.resourcemanager.utils.RMConfiguration._
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ModuleResourceRecordService.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ModuleResourceRecordService.scala
index 1d2f880..ba0f6ac 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ModuleResourceRecordService.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ModuleResourceRecordService.scala
@@ -17,20 +17,18 @@
package com.webank.wedatasphere.linkis.resourcemanager.service.metadata
import java.util
-import java.util.{Date, List}
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceRequestPolicy, ResourceSerializer}
import com.webank.wedatasphere.linkis.resourcemanager.dao.{EmMetaDataDao, EmResourceMetaDataDao}
import com.webank.wedatasphere.linkis.resourcemanager.domain.{EmMetaData, EmResourceMetaData, ModuleInfo, ModuleResourceRecord}
import com.webank.wedatasphere.linkis.resourcemanager.exception.RMErrorException
+import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceRequestPolicy, ResourceSerializer}
import org.json4s.DefaultFormats
import org.json4s.jackson.Serialization.{read, write}
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component
-import collection.JavaConversions._
import scala.collection.JavaConversions
/**
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ResourceLockService.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ResourceLockService.scala
index 881c5fd..e68a6f9 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ResourceLockService.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/ResourceLockService.scala
@@ -16,8 +16,6 @@
package com.webank.wedatasphere.linkis.resourcemanager.service.metadata
-import java.util.concurrent.TimeUnit
-
import com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.resourcemanager.dao.ResourceLockDao
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserConfiguration.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserConfiguration.scala
index 075929d..5c782fa 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserConfiguration.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserConfiguration.scala
@@ -20,7 +20,7 @@
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.protocol.CacheableProtocol
-import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfig, RequestQueryAppConfigWithGlobal, ResponseQueryConfig}
+import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfigWithGlobal, ResponseQueryConfig}
import com.webank.wedatasphere.linkis.resourcemanager.utils.RMConfiguration
import com.webank.wedatasphere.linkis.rpc.RPCMapCache
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserResourceRecordService.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserResourceRecordService.scala
index 93a0e9e..61b1506 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserResourceRecordService.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/metadata/UserResourceRecordService.scala
@@ -18,11 +18,11 @@
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
import com.webank.wedatasphere.linkis.resourcemanager.dao.UserResourceMetaDataDao
-import com.webank.wedatasphere.linkis.resourcemanager.domain.{UserModuleRecord, UserResourceMetaData}
+import com.webank.wedatasphere.linkis.resourcemanager.domain.UserResourceMetaData
import com.webank.wedatasphere.linkis.resourcemanager.event.notify.UserPreUsedEvent
import com.webank.wedatasphere.linkis.resourcemanager.exception.{RMErrorException, RMWarnException}
+import com.webank.wedatasphere.linkis.resourcemanager.{Resource, ResourceSerializer}
import org.json4s.DefaultFormats
import org.json4s.jackson.Serialization.{read, write}
import org.springframework.beans.factory.annotation.Autowired
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultResourceManager.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultResourceManager.scala
index 055c4a1..80df607 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultResourceManager.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultResourceManager.scala
@@ -26,8 +26,8 @@
import com.webank.wedatasphere.linkis.resourcemanager.domain._
import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope
import com.webank.wedatasphere.linkis.resourcemanager.event.notify.{ModuleRegisterEvent, _}
-import com.webank.wedatasphere.linkis.resourcemanager.schedule.EventGroupFactory
import com.webank.wedatasphere.linkis.resourcemanager.exception.{RMErrorException, RMWarnException}
+import com.webank.wedatasphere.linkis.resourcemanager.schedule.EventGroupFactory
import com.webank.wedatasphere.linkis.resourcemanager.service.metadata.{ModuleResourceRecordService, ResourceLockService, UserMetaData, UserResourceRecordService}
import com.webank.wedatasphere.linkis.resourcemanager.utils.RMConfiguration
import com.webank.wedatasphere.linkis.rpc.Sender
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultUserResourceManager.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultUserResourceManager.scala
index 799f84e..70e24bd 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultUserResourceManager.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/service/rm/DefaultUserResourceManager.scala
@@ -22,13 +22,10 @@
import com.webank.wedatasphere.linkis.resourcemanager.event.notify._
import com.webank.wedatasphere.linkis.resourcemanager.service.metadata.{ModuleResourceRecordService, UserResourceRecordService}
import com.webank.wedatasphere.linkis.resourcemanager.utils.RMUtils
-
import com.webank.wedatasphere.linkis.resourcemanager.{Resource, UserResourceManager}
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component
-import scala.collection.mutable
-
/**
* Created by johnnwang on 2018/9/15.
*/
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/RMUtils.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/RMUtils.scala
index 013faa0..8840911 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/RMUtils.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/RMUtils.scala
@@ -16,9 +16,7 @@
package com.webank.wedatasphere.linkis.resourcemanager.utils
-import com.webank.wedatasphere.linkis.common.exception.DWCException
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
-import org.slf4j.Logger
/**
* Created by shanhuang on 9/11/18.
diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala
index ac367f4..539cff0 100644
--- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala
+++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala
@@ -20,33 +20,39 @@
import com.fasterxml.jackson.core.JsonParseException
import com.webank.wedatasphere.linkis.common.conf.CommonVars
-import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.common.conf.Configuration.hadoopConfDir
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.resourcemanager.YarnResource
import com.webank.wedatasphere.linkis.resourcemanager.exception.{RMErrorException, RMFatalException, RMWarnException}
-import dispatch.{Http, as}
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.fs.Path
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.util.RMHAUtils
+import org.apache.http.client.methods.HttpGet
+import org.apache.http.impl.client.HttpClients
+import org.apache.http.util.EntityUtils
import org.json4s.JsonAST._
+import org.json4s.jackson.JsonMethods._
import org.json4s.{DefaultFormats, JValue}
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
-import scala.concurrent.{Await, ExecutionContext}
-import scala.concurrent.duration.Duration
/**
- * Created by shanhuang on 2018/9/24.
- */
+ * Created by shanhuang on 2018/9/24.
+ */
object YarnUtil extends Logging{
- private implicit val executor = ExecutionContext.global
- private var yarnConf: YarnConfiguration = _
- private var rm_web_address: String = CommonVars("wds.linkis.yarn.rm.web.address", "").getValue
- private var hadoop_version:String = "2.7.2"
+
implicit val format = DefaultFormats
+ private var yarnConf: YarnConfiguration = _
+
+ private var rm_web_address: String = CommonVars("wds.linkis.yarn.rm.web.address", "").getValue
+
+ private var hadoop_version:String = "2.7.2"
+
+ private val httpClient = HttpClients.createDefault()
+
def init() = {
if(StringUtils.isBlank(this.rm_web_address)){
yarnConf = new YarnConfiguration()
@@ -101,30 +107,31 @@
info(s"Resource Manager WebApp address: $rm_web_address.")
}
+ private def getResponseByUrl(url: String): JValue = {
+ val httpGet = new HttpGet(rm_web_address + "/ws/v1/cluster/" + url)
+ httpGet.addHeader("Accept", "application/json")
+ val response = httpClient.execute(httpGet)
+ parse(EntityUtils.toString(response.getEntity()))
+ }
+
private def getHadoopVersion():Unit = {
- val url = dispatch.url(rm_web_address) / "ws" / "v1" / "cluster" / "info"
- val future = Http(url > as.json4s.Json).map {resp =>
- val resourceManagerVersion = resp \ "clusterInfo" \ "resourceManagerVersion"
- info(s"Hadoop version is $resourceManagerVersion")
- hadoop_version = resourceManagerVersion.values.asInstanceOf[String]
- }
+ val resourceManagerVersion = getResponseByUrl("info") \ "clusterInfo" \ "resourceManagerVersion"
+
+ info(s"Hadoop version is $resourceManagerVersion")
+
+ hadoop_version = resourceManagerVersion.values.asInstanceOf[String]
}
def getQueueInfo(queueName: String): (YarnResource, YarnResource) = {
- val url = dispatch.url(rm_web_address) / "ws" / "v1" / "cluster" / "scheduler"
- url.setContentType("application/json", "UTF-8")
def getYarnResource(jValue: Option[JValue]): Option[YarnResource] = {
jValue.map(r => new YarnResource((r \ "memory").asInstanceOf[JInt].values.toLong * 1024l * 1024l, (r \ "vCores").asInstanceOf[JInt].values.toInt, 0, queueName))
}
def maxEffectiveHandle(queueValue: Option[JValue]): Option[YarnResource] = {
- val url = dispatch.url(rm_web_address) / "ws" / "v1" / "cluster" / "metrics"
- url.setContentType("application/json", "UTF-8")
- val totalResouceInfo = Http(url > as.json4s.Json).map { resp => ((resp \ "clusterMetrics" \ "totalMB").asInstanceOf[JInt].values.toLong, (resp \ "clusterMetrics" \ "totalVirtualCores").asInstanceOf[JInt].values.toLong) }
- val totalResouceInfoResponse = Await.result(totalResouceInfo, Duration.create(10, "seconds"))
-
+ val metrics = getResponseByUrl( "metrics")
+ val totalResouceInfoResponse = ((metrics \ "clusterMetrics" \ "totalMB").asInstanceOf[JInt].values.toLong, (metrics \ "clusterMetrics" \ "totalVirtualCores").asInstanceOf[JInt].values.toLong)
queueValue.map(r => {
val effectiveResource = (r \ "absoluteCapacity").asInstanceOf[JDecimal].values.toDouble- (r \ "absoluteUsedCapacity").asInstanceOf[JDecimal].values.toDouble
new YarnResource(math.floor(effectiveResource * totalResouceInfoResponse._1 * 1024l * 1024l/100).toLong, math.floor(effectiveResource * totalResouceInfoResponse._2/100).toInt, 0, queueName)
@@ -151,8 +158,7 @@
}
def getChildQueues(resp:JValue):JValue = {
val queues = resp \ "childQueues" \ "queue"
-
- if(queues != null && queues != JNull && queues != JNothing ) {
+ if(queues != null && queues != JNull && queues != JNothing && queues.children.nonEmpty) {
info(s"test queue:$queues")
queues
} else resp \ "childQueues"
@@ -183,57 +189,61 @@
def getChildQueuesOfCapacity(resp:JValue):JValue = resp \ "queues" \ "queue"
- val future = Http(url > as.json4s.Json).map {resp =>
+ def getResources(): (YarnResource, YarnResource) = {
+ val resp = getResponseByUrl("scheduler")
val schedulerType = (resp \ "scheduler" \ "schedulerInfo" \ "type").asInstanceOf[JString].values
if ("capacityScheduler".equals(schedulerType)) {
realQueueName = queueName
val childQueues = getChildQueuesOfCapacity(resp \ "scheduler" \ "schedulerInfo")
val queue = getQueueOfCapacity(childQueues)
- if(queue.isEmpty) {
+ if (queue.isEmpty) {
debug(s"cannot find any information about queue $queueName, response: " + resp)
- throw new RMWarnException(111006, s"queue $queueName is not exists in YARN.")
+ throw new RMWarnException(11006, s"queue $queueName is not exists in YARN.")
}
(maxEffectiveHandle(queue).get, getYarnResource(queue.map( _ \ "resourcesUsed")).get)
} else if ("fairScheduler".equals(schedulerType)) {
val childQueues = getChildQueues(resp \ "scheduler" \ "schedulerInfo" \ "rootQueue")
val queue = getQueue(childQueues)
- if(queue.isEmpty) {
+ if (queue.isEmpty) {
debug(s"cannot find any information about queue $queueName, response: " + resp)
- throw new RMWarnException(111006, s"queue $queueName is not exists in YARN.")
+ throw new RMWarnException(11006, s"queue $queueName is not exists in YARN.")
}
- (getYarnResource(queue.map( _ \ "maxResources")).get,
- getYarnResource(queue.map( _ \ "usedResources")).get)
+ (getYarnResource(queue.map(_ \ "maxResources")).get,
+ getYarnResource(queue.map(_ \ "usedResources")).get)
} else {
debug(s"only support fairScheduler or capacityScheduler, schedulerType: $schedulerType , response: " + resp)
- throw new RMWarnException(111006, s"only support fairScheduler or capacityScheduler, schedulerType: $schedulerType")
+ throw new RMWarnException(11006, s"only support fairScheduler or capacityScheduler, schedulerType: $schedulerType")
}
}
- Utils.tryCatch(Await.result(future, Duration.Inf))( t => {
- if((t.getCause.isInstanceOf[JsonParseException] && t.getCause.getMessage.contains("This is standby RM"))
+
+ Utils.tryCatch(getResources())(t => {
+ if ((t.getCause.isInstanceOf[JsonParseException] && t.getCause.getMessage.contains("This is standby RM"))
|| t.getCause.isInstanceOf[ConnectException]) {
reloadRMWebAddress()
getQueueInfo(queueName)
- } else throw new RMErrorException(111006, "Get the Yarn queue information exception.(获取Yarn队列信息异常)", t)
+ } else throw new RMErrorException(11006, "Get the Yarn queue information exception" +
+ ".(获取Yarn队列信息异常)", t)
})
}
def getApplicationsInfo(queueName: String): Array[YarnAppInfo] = {
- val url = dispatch.url(rm_web_address) / "ws" / "v1" / "cluster" / "apps"
- url.setContentType("application/json", "UTF-8")
+
def getYarnResource(jValue: Option[JValue]): Option[YarnResource] = {
jValue.map(r => new YarnResource((r \ "allocatedMB").asInstanceOf[JInt].values.toLong * 1024l * 1024l, (r \ "allocatedVCores").asInstanceOf[JInt].values.toInt, 0, queueName))
}
val realQueueName = "root." + queueName
- val future = Http(url > as.json4s.Json).map {resp =>
+
+ def getAppInfos(): Array[YarnAppInfo] = {
+ val resp = getResponseByUrl("apps")
resp \ "apps" \ "app" match {
case JArray(apps) =>
val appInfoBuffer = new ArrayBuffer[YarnAppInfo]()
apps.foreach { app =>
val yarnQueueName = (app \ "queue").asInstanceOf[JString].values
val state = (app \ "state").asInstanceOf[JString].values
- if(yarnQueueName == realQueueName && (state == "RUNNING" || state == "ACCEPTED")){
+ if (yarnQueueName == realQueueName && (state == "RUNNING" || state == "ACCEPTED")) {
val appInfo = new YarnAppInfo(
(app \ "id").asInstanceOf[JString].values,
(app \ "user").asInstanceOf[JString].values,
@@ -248,12 +258,13 @@
case JNull | JNothing => new Array[YarnAppInfo](0)
}
}
- Utils.tryCatch(Await.result(future, Duration.Inf))( t => {
- if((t.getCause.isInstanceOf[JsonParseException] && t.getCause.getMessage.contains("This is standby RM"))
+
+ Utils.tryCatch(getAppInfos())(t => {
+ if ((t.getCause.isInstanceOf[JsonParseException] && t.getCause.getMessage.contains("This is standby RM"))
|| t.getCause.isInstanceOf[ConnectException]) {
reloadRMWebAddress()
getApplicationsInfo(queueName)
- } else throw new RMErrorException(111006, "Get the Yarn Application information exception.(获取Yarn Application信息异常)", t)
+ } else throw new RMErrorException(11006, "Get the Yarn Application information exception.(获取Yarn Application信息异常)", t)
})
}
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/NotifyRMEventSerializerTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/NotifyRMEventSerializerTest.scala
deleted file mode 100644
index 0f1d9cf..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/NotifyRMEventSerializerTest.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.resourcemanager.notify
-
-import com.webank.wedatasphere.linkis.common.ServiceInstance
-import com.webank.wedatasphere.linkis.resourcemanager.{MemoryResource, ResourceRequestPolicy}
-import com.webank.wedatasphere.linkis.resourcemanager.domain._
-import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope
-import com.webank.wedatasphere.linkis.resourcemanager.event.notify._
-import org.json4s.DefaultFormats
-import org.json4s.jackson.Serialization.read
-import org.json4s.jackson.Serialization.write
-
-import scala.collection.mutable.ArrayBuffer
-
-object NotifyRMEventSerializerTest {
-
- implicit val formats = DefaultFormats + NotifyRMEventSerializer
- var event: NotifyRMEvent = _
- var serialized: String = _
-
- def main(args: Array[String]): Unit = {
-
- }
-
- private def checkResult = {
- serialized = write(event)
- System.out.println(serialized)
- val recovered = read[NotifyRMEvent](serialized)
- System.out.println(write(recovered))
- }
-}
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ResourceTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ResourceTest.scala
deleted file mode 100644
index 92b2821..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ResourceTest.scala
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.resourcemanager.notify
-
-import com.webank.wedatasphere.linkis.resourcemanager.YarnResource
-
-object ResourceTest {
- def main(args: Array[String]): Unit = {
-
- }
-}
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperLockTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperLockTest.scala
deleted file mode 100644
index 184a248..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperLockTest.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.resourcemanager.notify
-
-object ZookeeperLockTest {
-
- def main(args: Array[String]): Unit = {
-
- val thread1 = new TestThread("a")
- val thread2 = new TestThread("b")
- val thread3 = new TestThread("c")
- val thread4 = new TestThread("d")
- val thread5 = new TestThread("e")
-
- thread1.start()
- thread2.start()
- thread3.start()
- thread4.start()
- thread5.start()
-
- }
-
- class TestThread(name: String) extends Thread {
- override def run(): Unit = {
- val lock = ZookeeperDistributedLock("/dwc_test", "test_lock")
- try {
- lock.acquire()
- System.out.println(name + " get lock")
- Thread.sleep(3000)
- } finally {
- lock.release()
- System.out.println(name + " release lock")
- }
- }
- }
-
-}
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperPubTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperPubTest.scala
deleted file mode 100644
index 2f2ecda..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperPubTest.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.resourcemanager.notify
-
-import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope
-import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope.Instance
-import com.webank.wedatasphere.linkis.resourcemanager.event.notify.DefaultNotifyRMEvent
-
-object ZookeeperPubTest {
- def main(args: Array[String]): Unit = {
- val threadPub = new PubThread
- threadPub.start()
- }
-
-}
-
-class PubThread extends Thread {
- val modules = Array("a", "b", "c", "d", "e", "f")
- val scopes = Array(EventScope.Instance, EventScope.Other, EventScope.Service, EventScope.User)
- val publisher = NotifyRMEventPublisher("test_queue")
-
- override def run(): Unit = {
- (0 to 100).foreach { i =>
- publisher.publish(new DefaultNotifyRMEvent(i.toString, modules(i % modules.length), scopes(i % scopes.length)))
- Thread.sleep(500)
- }
- }
-}
\ No newline at end of file
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperQueueTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperQueueTest.scala
deleted file mode 100644
index 61d0787..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperQueueTest.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.resourcemanager.notify
-
-import com.webank.wedatasphere.linkis.resourcemanager.event.EventScope.{EventScope => _, _}
-import com.webank.wedatasphere.linkis.resourcemanager.event._
-import com.webank.wedatasphere.linkis.resourcemanager.event.notify.{DefaultNotifyRMEvent, NotifyRMEvent}
-import com.webank.wedatasphere.linkis.resourcemanager.schedule.{EventSchedulerContextImpl, EventSchedulerImpl}
-import org.json4s.DefaultFormats
-import org.json4s.jackson.Serialization.{read, write}
-
-object ZookeeperQueueTest {
-
- def main(args: Array[String]): Unit = {
-
- val threadSub = new Thread {
- val subscriber = NotifyRMEventSubscriber("test_queue", new EventSchedulerImpl(new EventSchedulerContextImpl(1)))
-
- override def run(): Unit = {
- subscriber.start()
- Thread.sleep(60000)
- subscriber.stop()
- }
- }
-
- val threadSub1 = new Thread {
- val subscriber = NotifyRMEventSubscriber("test_queue", new EventSchedulerImpl(new EventSchedulerContextImpl(1)))
-
- override def run(): Unit = {
- subscriber.start()
- Thread.sleep(60000)
- subscriber.stop()
- }
- }
-
- val threadPub = new Thread {
- val publisher = NotifyRMEventPublisher("test_queue")
-
- override def run(): Unit = {
- (0 to 1).foreach { i =>
- publisher.publish(new DefaultNotifyRMEvent(i.toString, "test", Instance))
- }
-
- }
- }
-
- threadSub1.start()
- threadPub.start()
- threadSub.start()
-
- }
-}
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperSubTest.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperSubTest.scala
deleted file mode 100644
index 9cb3a5a..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/ZookeeperSubTest.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.resourcemanager.notify
-
-import com.webank.wedatasphere.linkis.resourcemanager.schedule.{EventSchedulerContextImpl, EventSchedulerImpl}
-
-object ZookeeperSubTest {
- def main(args: Array[String]): Unit = {
- val threadSub = new SubThread
- val threadSub1 = new SubThread
-
- threadSub.start()
- threadSub1.start()
- }
-}
-
-class SubThread extends Thread {
- val subscriber = NotifyRMEventSubscriber("test_queue", new EventSchedulerImpl(new EventSchedulerContextImpl(1)))
-
- override def run(): Unit = {
- subscriber.start()
- Thread.sleep(60000)
- subscriber.stop()
- }
-}
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestMain.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestMain.scala
deleted file mode 100644
index 57af396..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestMain.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.test
-
-
-import com.webank.wedatasphere.linkis.resourcemanager.domain.{ModuleInfo, ModuleInfoSerializer, ModuleInstanceSerializer, ModuleResourceInfo}
-import com.webank.wedatasphere.linkis.resourcemanager.event.notify.{NotifyRMEvent, NotifyRMEventSerializer}
-import com.webank.wedatasphere.linkis.resourcemanager.{LoadResource, ResourceRequestPolicy, ResourceSerializer, YarnResource}
-import org.json4s.JsonAST.JObject
-import org.json4s.{CustomSerializer, DefaultFormats, Extraction}
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods._
-import org.json4s.jackson.Serialization._
-
-
-/**
- * Created by johnnwang on 9/15/18.
- */
-class ResultResource
-
-case class NotEnoughResource(reason: String = null) extends ResultResource
-
-class AvailableResource(val ticketId: String) extends ResultResource
-
-case class UserResultResource(ticketId: String, user: String) extends ResultResource
-
-object ResultResourceSerializer1 extends CustomSerializer[ResultResource](implicit formats => ( {
- case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => new AvailableResource(ticketId.extract[String])
-}, {
- case r: AvailableResource => ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId)))
-}))
-
-object ResultResourceSerializer2 extends CustomSerializer[ResultResource](implicit formats => ( {
- case JObject(List(("NotEnoughResource", JObject(List(("reason", reason)))))) => NotEnoughResource(reason.extract[String])
- // case JObject(List(("ticketId", ticketId))) => new AvailableResource(ticketId.extract[String])
- case JObject(List(("UserResultResource", JObject(List(("ticketId", ticketId), ("user", user)))))) =>
- UserResultResource(ticketId.extract[String], user.extract[String])
-}, {
- case r: NotEnoughResource => ("NotEnoughResource", ("reason", Extraction.decompose(r.reason)))
- case r: AvailableResource => ("ticketId", Extraction.decompose(r.ticketId))
- case r: UserResultResource => ("UserResultResource", ("ticketId", r.ticketId) ~ ("user", r.user))
-}))
-
-object TestMain {
- implicit val formats = DefaultFormats + ResultResourceSerializer1 + ResourceSerializer + ModuleInfoSerializer + ModuleInstanceSerializer + NotifyRMEventSerializer
-
- //implicit val formats = DefaultFormats + ResourceSerializer + ModuleResourceInfoSerializer + ResultResourceSerializer1 + ModuleInstanceSerializer + ModuleInfoSerializer
- def main(args: Array[String]): Unit = {
- val yarnResource1 = new YarnResource(100, 10, 0, "ide")
- val yarnResource2 = new YarnResource(1, 1, 0, "ide")
- println(yarnResource1 > yarnResource2)
- }
-}
diff --git a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestResource.scala b/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestResource.scala
deleted file mode 100644
index 7044544..0000000
--- a/resourceManager/resourcemanagerserver/src/test/scala/com/webank/test/TestResource.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.test
-
-import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Utils}
-import com.webank.wedatasphere.linkis.resourcemanager.utils.YarnUtil
-import com.webank.wedatasphere.linkis.resourcemanager.{DriverAndYarnResource, LoadInstanceResource, Resource, YarnResource}
-
-/**
- * Created by johnnwang on 2018/12/29.
- */
-object TestResource {
-
- def main(args: Array[String]): Unit = {
- val info = YarnUtil.getApplicationsInfo("q01")
- print(info)
- }
-
- // def main(args: Array[String]): Unit = {
- // val resource1 = new DriverAndYarnResource(new LoadInstanceResource(100,100,0),
- // new YarnResource(101440 * 1024l * 1024l,50,2,"q05"))
- // val resource2 = new DriverAndYarnResource(new LoadInstanceResource(50,50,0),
- // new YarnResource(103424 * 1024l * 1024l,2,0,"q05"))
- //
- // println((resource1 - resource2).toString)
- // }
- // def minus(resource1:Resource,resource2:Resource) = {
- // resource1 - resource2
- // }
-}
diff --git a/storage/pesIO/io-engine/pom.xml b/storage/pesIO/io-engine/pom.xml
index 718e279..2daa412 100644
--- a/storage/pesIO/io-engine/pom.xml
+++ b/storage/pesIO/io-engine/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/storage/pesIO/io-enginemanager/pom.xml b/storage/pesIO/io-enginemanager/pom.xml
index 8d58562..10bd4ca 100644
--- a/storage/pesIO/io-enginemanager/pom.xml
+++ b/storage/pesIO/io-enginemanager/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/storage/pesIO/io-entrance/pom.xml b/storage/pesIO/io-entrance/pom.xml
index 233bb48..a5c66f6 100644
--- a/storage/pesIO/io-entrance/pom.xml
+++ b/storage/pesIO/io-entrance/pom.xml
@@ -20,7 +20,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/storage/storage/pom.xml b/storage/storage/pom.xml
index 19302dd..221b845 100644
--- a/storage/storage/pom.xml
+++ b/storage/storage/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
@@ -52,6 +52,10 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
@@ -70,6 +74,12 @@
<groupId>com.monitorjbl</groupId>
<artifactId>xlsx-streamer</artifactId>
<version>1.2.1</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>jackson-databind</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java
index 2351ad2..a207bc8 100644
--- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java
+++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/BuildFactory.java
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.storage.factory;
import com.webank.wedatasphere.linkis.common.io.Fs;
-import com.webank.wedatasphere.linkis.common.io.FsPath;
/**
* Created by johnnwang on 10/17/18.
diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java
index 2b22b41..d31d235 100644
--- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java
+++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/factory/impl/BuildLocalFileSystem.java
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.storage.factory.impl;
import com.webank.wedatasphere.linkis.common.io.Fs;
-import com.webank.wedatasphere.linkis.common.io.FsPath;
import com.webank.wedatasphere.linkis.storage.factory.BuildFactory;
import com.webank.wedatasphere.linkis.storage.fs.FileSystem;
import com.webank.wedatasphere.linkis.storage.fs.impl.LocalFileSystem;
diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java
index de51915..f361006 100644
--- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java
+++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/FileSystem.java
@@ -22,7 +22,6 @@
import java.io.File;
import java.io.IOException;
-import java.util.Map;
/**
* Created by johnnwang on 10/15/18.
diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java
index d14ab08..b7a0060 100644
--- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java
+++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/fs/impl/LocalFileSystem.java
@@ -17,8 +17,6 @@
package com.webank.wedatasphere.linkis.storage.fs.impl;
import com.webank.wedatasphere.linkis.common.io.FsPath;
-import com.webank.wedatasphere.linkis.common.io.FsReader;
-import com.webank.wedatasphere.linkis.common.io.FsWriter;
import com.webank.wedatasphere.linkis.common.utils.Utils;
import com.webank.wedatasphere.linkis.storage.domain.FsPathListWithError;
import com.webank.wedatasphere.linkis.storage.exception.StorageWarnException;
diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java
index 6073004..323a1d8 100644
--- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java
+++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineReader.java
@@ -19,7 +19,6 @@
import com.webank.wedatasphere.linkis.common.io.FsReader;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
/**
diff --git a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java
index ababb0d..a6e9d1a 100644
--- a/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java
+++ b/storage/storage/src/main/java/com/webank/wedatasphere/linkis/storage/pipeline/PipelineWriter.java
@@ -18,7 +18,6 @@
import com.webank.wedatasphere.linkis.common.io.FsWriter;
-import java.io.IOException;
import java.util.Map;
/**
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala
index cc9868b..6b7aa39 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/FSFactory.scala
@@ -21,7 +21,6 @@
import com.webank.wedatasphere.linkis.storage.exception.StorageFatalException
import com.webank.wedatasphere.linkis.storage.factory.BuildFactory
import com.webank.wedatasphere.linkis.storage.utils.{StorageConfiguration, StorageUtils}
-import net.sf.cglib.proxy.Enhancer
/**
* Created by johnnwang on 10/17/18.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/CSVFsWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/CSVFsWriter.scala
index 66dc4b1..15fdef9 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/CSVFsWriter.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/CSVFsWriter.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.storage.csv
-import java.io.InputStream
+import java.io.OutputStream
import com.webank.wedatasphere.linkis.common.io.FsWriter
@@ -26,14 +26,9 @@
abstract class CSVFsWriter extends FsWriter {
val charset: String
val separator: String
- var isLastRow: Boolean = false
-
- def setIsLastRow(value: Boolean): Unit
-
- def getCSVStream: InputStream
}
object CSVFsWriter {
- def getCSVFSWriter(charset: String, separator: String): CSVFsWriter = new StorageCSVWriter(charset, separator)
+ def getCSVFSWriter(charset: String, separator: String, outputStream: OutputStream): CSVFsWriter = new StorageCSVWriter(charset, separator, outputStream)
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/StorageCSVWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/StorageCSVWriter.scala
index d903c10..5a08674 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/StorageCSVWriter.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/csv/StorageCSVWriter.scala
@@ -17,79 +17,60 @@
package com.webank.wedatasphere.linkis.storage.csv
import java.io._
-import java.util
-import java.util.Collections
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.storage.domain.DataType
import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord}
+import org.apache.commons.io.IOUtils
/**
* Created by johnnwang on 2018/11/12.
*/
-class StorageCSVWriter(charsetP: String, separatorP: String) extends CSVFsWriter with Logging {
-
- override val charset: String = charsetP
- override val separator: String = separatorP
+class StorageCSVWriter(val charset: String, val separator: String, val outputStream: OutputStream) extends CSVFsWriter with Logging {
private val delimiter = separator match {
case "," => ','
- case _ =>'\t'
+ case _ => '\t'
}
- private val cSVWriter = new util.ArrayList[InputStream]()
- private var stream: SequenceInputStream = _
- private val buffer: StringBuilder = new StringBuilder(40000)
- private var counter: Int = _
-
- override def setIsLastRow(value: Boolean): Unit = {}
-
- def collectionInputStream(content: Array[String]): Unit = {
- content.foreach(f => counter += f.length)
- counter += content.size
- if (counter >= 40000) {
- cSVWriter.add(new ByteArrayInputStream(buffer.toString().getBytes(charset)))
- buffer.clear()
- content.indices.map {
- case 0 => content(0)
- case i => delimiter + content(i)
- }.foreach(buffer.append)
- buffer.append("\n")
- counter = buffer.length
- } else {
- content.indices.map {
- case 0 => content(0)
- case i => delimiter + content(i)
- }.foreach(buffer.append)
- buffer.append("\n")
- counter = buffer.length
- }
- }
+ private val buffer: StringBuilder = new StringBuilder(50000)
@scala.throws[IOException]
override def addMetaData(metaData: MetaData): Unit = {
val head = metaData.asInstanceOf[TableMetaData].columns.map(_.columnName)
- collectionInputStream(head)
+ write(head)
+ //IOUtils.write(compact(head).getBytes(charset),outputStream)
+ }
+
+ private def compact(row: Array[String]): String = {
+ val tmp = row.foldLeft("")((l, r) => l + delimiter + r)
+ tmp.substring(1, tmp.length) + "\n"
+ }
+
+ private def write(row: Array[String]) = {
+ val cotent: String = compact(row)
+ if (buffer.length + cotent.length > 49500) {
+ IOUtils.write(buffer.toString().getBytes(charset), outputStream)
+ buffer.clear()
+ }
+ buffer.append(cotent)
}
@scala.throws[IOException]
override def addRecord(record: Record): Unit = {
- val body = record.asInstanceOf[TableRecord].row.map(f => if (f != null) f.toString else DataType.NULL_VALUE)
- collectionInputStream(body)
+ val body = record.asInstanceOf[TableRecord].row.map(_.toString) //read时候进行null替换等等
+ write(body)
+ //IOUtils.write(compact(body).getBytes(charset),outputStream)
}
- override def flush(): Unit = {}
+ override def flush(): Unit = {
+ IOUtils.write(buffer.toString().getBytes(charset), outputStream)
+ buffer.clear()
+ }
override def close(): Unit = {
- if (stream != null) stream.close()
+ flush()
+ IOUtils.closeQuietly(outputStream)
}
- override def getCSVStream: InputStream = {
- cSVWriter.add(new ByteArrayInputStream(buffer.toString().getBytes(charset)))
- buffer.clear()
- counter = 0 //Subsequent move to flush(后续挪到flush)
- stream = new SequenceInputStream(Collections.enumeration(cSVWriter))
- stream
- }
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala
index eb6c32b..01153e4 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/domain/DataType.scala
@@ -17,8 +17,8 @@
package com.webank.wedatasphere.linkis.storage.domain
import java.sql.{Date, Timestamp}
-import com.webank.wedatasphere.linkis.common.utils.Utils
-import com.webank.wedatasphere.linkis.common.utils.Logging
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
/**
* Created by johnnwang on 10/17/18.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/ExcelFsWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/ExcelFsWriter.scala
index 6c6afbb..5901440 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/ExcelFsWriter.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/ExcelFsWriter.scala
@@ -16,21 +16,20 @@
package com.webank.wedatasphere.linkis.storage.excel
+import java.io.OutputStream
+
import com.webank.wedatasphere.linkis.common.io.FsWriter
-import org.apache.poi.ss.usermodel.Workbook
/**
* Created by johnnwang on 2018/11/12.
*/
-abstract class ExcelFsWriter extends FsWriter{
- val charset:String
- val sheetName:String
- val dateFormat:String
-
- def getWorkBook: Workbook
+abstract class ExcelFsWriter extends FsWriter {
+ val charset: String
+ val sheetName: String
+ val dateFormat: String
}
-object ExcelFsWriter{
- def getExcelFsWriter(charset:String,sheetName:String,dateFormat:String):ExcelFsWriter = new StorageExcelWriter(charset,sheetName,dateFormat)
+object ExcelFsWriter {
+ def getExcelFsWriter(charset: String, sheetName: String, dateFormat: String, outputStream: OutputStream): ExcelFsWriter = new StorageExcelWriter(charset, sheetName, dateFormat, outputStream: OutputStream)
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/StorageExcelWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/StorageExcelWriter.scala
index 69c9d8e..6b3ba72 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/StorageExcelWriter.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/excel/StorageExcelWriter.scala
@@ -16,14 +16,13 @@
package com.webank.wedatasphere.linkis.storage.excel
-import java.io.IOException
-import java.text.SimpleDateFormat
+import java.io._
import java.util
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
-import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.storage.domain.DataType
import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord}
+import org.apache.commons.io.IOUtils
import org.apache.poi.ss.usermodel._
import org.apache.poi.xssf.streaming.{SXSSFSheet, SXSSFWorkbook}
@@ -32,10 +31,7 @@
/**
* Created by johnnwang on 2018/11/12.
*/
-class StorageExcelWriter(charsetP: String, sheetNameP: String, dateFormatP: String) extends ExcelFsWriter {
- override val charset: String = charsetP
- override val sheetName: String = sheetNameP
- override val dateFormat: String = dateFormatP
+class StorageExcelWriter(val charset: String, val sheetName: String, val dateFormat: String, val outputStream: OutputStream) extends ExcelFsWriter {
private var workBook: SXSSFWorkbook = _
private var sheet: SXSSFSheet = _
@@ -43,7 +39,10 @@
private var types: Array[DataType] = _
private var rowPoint = 0
private var columnCounter = 0
- private val styles = new util.HashMap[String,CellStyle]()
+ private val styles = new util.HashMap[String, CellStyle]()
+ private var isFlush = true
+ private val os = new ByteArrayOutputStream()
+ private var is: ByteArrayInputStream = _
def init = {
workBook = new SXSSFWorkbook()
@@ -61,8 +60,8 @@
}
def getWorkBook: Workbook = {
- //Adaptive column width(自适应列宽)
- sheet.trackAllColumnsForAutoSizing();
+ //自适应列宽
+ sheet.trackAllColumnsForAutoSizing()
for (elem <- 0 to columnCounter) {
sheet.autoSizeColumn(elem)
}
@@ -78,13 +77,13 @@
style
}
- def getCellStyle(dataType: DataType):CellStyle = {
+ def getCellStyle(dataType: DataType): CellStyle = {
val style = styles.get(dataType.typeName)
if (style == null) {
val newStyle = createCellStyle(dataType)
- styles.put(dataType.typeName,newStyle)
+ styles.put(dataType.typeName, newStyle)
newStyle
- } else{
+ } else {
style
}
}
@@ -109,16 +108,14 @@
@scala.throws[IOException]
override def addRecord(record: Record): Unit = {
- // TODO: Do you need to replace the null value?(是否需要替换null值)
+ // TODO: 是否需要替换null值
val tableBody = sheet.createRow(rowPoint)
var colunmPoint = 0
val excelRecord = record.asInstanceOf[TableRecord].row
for (elem <- excelRecord) {
val cell = tableBody.createCell(colunmPoint)
val dataType = types.apply(colunmPoint)
- dataType.toString match {
- case _ =>if(elem !=null) cell.setCellValue(elem.toString) else cell.setCellValue(DataType.NULL_VALUE)
- }
+ cell.setCellValue(elem.toString) //read时候进行null替换等等
cell.setCellStyle(getCellStyle(dataType))
colunmPoint += 1
}
@@ -126,10 +123,28 @@
}
- override def flush(): Unit = {}
+ override def flush(): Unit = {
+ getWorkBook.write(os)
+ val content: Array[Byte] = os.toByteArray
+ is = new ByteArrayInputStream(content)
+ val buffer: Array[Byte] = new Array[Byte](1024)
+ var bytesRead: Int = 0
+ while (isFlush) {
+ bytesRead = is.read(buffer, 0, 1024)
+ if (bytesRead == -1) {
+ isFlush = false
+ } else {
+ outputStream.write(buffer, 0, bytesRead)
+ }
+ }
+ }
override def close(): Unit = {
- Utils.tryFinally(if (workBook != null) workBook.close())()
+ if (isFlush) flush()
+ IOUtils.closeQuietly(outputStream)
+ IOUtils.closeQuietly(is)
+ IOUtils.closeQuietly(os)
+ IOUtils.closeQuietly(workBook)
}
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala
index 5697b3d..618da1b 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/DefaultResultSetFactory.scala
@@ -18,12 +18,12 @@
import java.util
-import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet
+import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
-import com.webank.wedatasphere.linkis.storage.exception.{StorageErrorException, StorageFatalException, StorageWarnException}
+import com.webank.wedatasphere.linkis.storage.exception.{StorageErrorException, StorageWarnException}
import com.webank.wedatasphere.linkis.storage.utils.{StorageConfiguration, StorageUtils}
import org.apache.commons.lang.StringUtils
@@ -75,6 +75,7 @@
val resultSetType = Dolphin.getType(inputStream)
if(StringUtils.isEmpty(resultSetType)) throw new StorageWarnException(51000, s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)")
Utils.tryQuietly(inputStream.close())
+ Utils.tryQuietly(fs.close())
getResultSetByType(resultSetType)
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala
index 511ab9d..ca8f708 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSet.scala
@@ -16,8 +16,8 @@
package com.webank.wedatasphere.linkis.storage.resultset
+import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet
import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
-import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer, ResultSet}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
import com.webank.wedatasphere.linkis.storage.utils.StorageConfiguration
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala
index ee15588..e0fece3 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetReader.scala
@@ -20,7 +20,7 @@
import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSet, ResultSetReader}
import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
-import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
import com.webank.wedatasphere.linkis.storage.exception.StorageWarnException
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala
index be6a4f1..b5f9218 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/StorageResultSetWriter.scala
@@ -17,14 +17,12 @@
package com.webank.wedatasphere.linkis.storage.resultset
import java.io.{IOException, OutputStream}
-import java.util
import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSerializer, ResultSet, ResultSetWriter}
import com.webank.wedatasphere.linkis.common.io.{Fs, FsPath, MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
-import com.webank.wedatasphere.linkis.storage.exception.StorageErrorException
import com.webank.wedatasphere.linkis.storage.utils.{FileSystemUtils, StorageUtils}
import scala.collection.mutable.ArrayBuffer
@@ -55,6 +53,8 @@
private var proxyUser:String = StorageUtils.getJvmUser
+ def getMetaData: MetaData = rMetaData
+
def setProxyUser(proxyUser:String): Unit = {
this.proxyUser = proxyUser
}
@@ -72,7 +72,7 @@
fs = FSFactory.getFsByProxyUser(storePath,proxyUser)
fs.init(null)
FileSystemUtils.createNewFile(storePath, proxyUser,true)
- outputStream = fs.write(storePath, false)
+ outputStream = fs.write(storePath, true)
info(s"Succeed to create a new file:$storePath")
}
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala
index 99a6d0e..a2f16c7 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/html/HtmlResultSet.scala
@@ -18,8 +18,8 @@
import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer}
import com.webank.wedatasphere.linkis.storage.resultset.txt.{TextResultDeserializer, TextResultSerializer}
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet}
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
/**
* Created by johnnwang on 2018/12/10.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala
index 84b9ce9..bc9ecd1 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/io/IOResultSerializer.scala
@@ -16,8 +16,8 @@
package com.webank.wedatasphere.linkis.storage.resultset.io
-import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.common.io.resultset.ResultSerializer
+import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
import org.apache.commons.codec.binary.Base64
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala
index 113d0e9..ce8c4ec 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/picture/PictureResultSet.scala
@@ -17,9 +17,9 @@
package com.webank.wedatasphere.linkis.storage.resultset.picture
import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer}
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
-import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet}
import com.webank.wedatasphere.linkis.storage.resultset.txt.{TextResultDeserializer, TextResultSerializer}
+import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet}
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
/**
* Created by johnnwang on 2018/12/10.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala
index 9018d96..63b2114 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultDeserializer.scala
@@ -17,8 +17,8 @@
package com.webank.wedatasphere.linkis.storage.resultset.txt
import com.webank.wedatasphere.linkis.common.io.resultset.ResultDeserializer
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
/**
* Created by johnnwang on 10/20/18.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala
index 232fce7..940cf28 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSerializer.scala
@@ -16,12 +16,10 @@
package com.webank.wedatasphere.linkis.storage.resultset.txt
-import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.common.io.resultset.ResultSerializer
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
+import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
import com.webank.wedatasphere.linkis.storage.domain.Dolphin
-
-import scala.tools.scalap.scalax.util.StringUtil
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
/**
* Created by johnnwang on 10/20/18.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala
index 0a0bbc0..ae3b19b 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/resultset/txt/TextResultSet.scala
@@ -17,8 +17,8 @@
package com.webank.wedatasphere.linkis.storage.resultset.txt
import com.webank.wedatasphere.linkis.common.io.resultset.{ResultDeserializer, ResultSerializer}
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, StorageResultSet}
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
/**
* Created by johnnwang on 10/20/18.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/ScriptFsWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/ScriptFsWriter.scala
index 4df2ac5..43b3a32 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/ScriptFsWriter.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/ScriptFsWriter.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.storage.script
-import java.io.OutputStream
+import java.io.{InputStream, OutputStream}
import com.webank.wedatasphere.linkis.common.io.{FsPath, FsWriter, MetaData}
import com.webank.wedatasphere.linkis.storage.LineRecord
@@ -28,25 +28,27 @@
val path: FsPath
val charset: String
- val outputStream: OutputStream
+
+ def getInputStream(): InputStream
}
object ScriptFsWriter {
- def getScriptFsWriter(path: FsPath, charset: String,outputStream: OutputStream): ScriptFsWriter = new StorageScriptFsWriter(path, charset,outputStream)
+ def getScriptFsWriter(path: FsPath, charset: String, outputStream: OutputStream = null): ScriptFsWriter =
+ new StorageScriptFsWriter(path, charset, outputStream)
}
object ParserFactory {
- def listParsers(): Array[Parser] = Array(PYScriptParser(), QLScriptParser(),ScalaScriptParser())
+ def listParsers(): Array[Parser] = Array(PYScriptParser(), QLScriptParser(), ScalaScriptParser())
}
object Compaction {
- def listCompactions(): Array[Compaction] = Array(PYScriptCompaction(),QLScriptCompaction(),ScalaScriptCompaction())
+ def listCompactions(): Array[Compaction] = Array(PYScriptCompaction(), QLScriptCompaction(), ScalaScriptCompaction())
}
trait Parser {
- def prefixConf:String
+ def prefixConf: String
def prefix: String
@@ -57,7 +59,7 @@
trait Compaction {
- def prefixConf:String
+ def prefixConf: String
def prefix: String
@@ -79,5 +81,5 @@
class ScriptRecord(line: String) extends LineRecord(line)
//definition variable; specialConfiguration ;runConfiguration; startUpConfiguration;
-case class Variable(sortParent:String,sort:String,key: String, value: String)
+case class Variable(sortParent: String, sort: String, key: String, value: String)
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala
index 2c88314..81bc6a0 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/VariableParser.scala
@@ -16,12 +16,8 @@
package com.webank.wedatasphere.linkis.storage.script
-import java.io.{FileInputStream, FileOutputStream}
import java.util
-import com.google.gson.{Gson, GsonBuilder}
-import com.webank.wedatasphere.linkis.common.io.FsPath
-
import scala.collection.mutable.ArrayBuffer
/**
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/PYScriptCompaction.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/PYScriptCompaction.scala
index 9803d9c..0b2f4a1 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/PYScriptCompaction.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/PYScriptCompaction.scala
@@ -23,7 +23,7 @@
override def belongTo(suffix: String): Boolean = {
suffix match {
- case "python"|"py" => true
+ case "python"|"py"|"sh" => true
case _ => false
}
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala
index 6e98d17..a9c0f7a 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/compaction/QLScriptCompaction.scala
@@ -25,6 +25,7 @@
suffix match {
case "sql" => true
case "hql" => true
+ case "jdbc" => true
case _ => false
}
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/reader/StorageScriptFsReader.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/reader/StorageScriptFsReader.scala
index b3e826c..f126753 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/reader/StorageScriptFsReader.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/reader/StorageScriptFsReader.scala
@@ -21,6 +21,7 @@
import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
import com.webank.wedatasphere.linkis.storage.script._
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import org.apache.commons.io.IOUtils
import scala.collection.mutable.ArrayBuffer
@@ -28,14 +29,10 @@
/**
* Created by johnnwang on 2018/10/23.
*/
-class StorageScriptFsReader(pathP: FsPath, charsetP: String, inputStreamP: InputStream) extends ScriptFsReader {
+class StorageScriptFsReader(val path: FsPath, val charset: String, val inputStream: InputStream) extends ScriptFsReader {
- override val path: FsPath = pathP
- override val charset: String = charsetP
-
- private val inputStream: InputStream = inputStreamP
private var inputStreamReader: InputStreamReader = _
- private var BufferedReader: BufferedReader = _
+ private var bufferedReader: BufferedReader = _
private var metadata: ScriptMetaData = _
@@ -47,7 +44,7 @@
if (metadata == null) throw new IOException("Must read metadata first(必须先读取metadata)")
val record = new ScriptRecord(lineText)
- lineText = BufferedReader.readLine()
+ lineText = bufferedReader.readLine()
record
}
@@ -55,10 +52,10 @@
override def getMetaData: MetaData = {
if (metadata == null) init()
val parser = ParserFactory.listParsers().filter(p => p.belongTo(StorageUtils.pathToSuffix(path.getPath)))
- lineText = BufferedReader.readLine()
- while (hasNext && parser.length > 0 && isMetadata(lineText, parser(0).prefix,parser(0).prefixConf)) {
+ lineText = bufferedReader.readLine()
+ while (hasNext && parser.length > 0 && isMetadata(lineText, parser(0).prefix, parser(0).prefixConf)) {
variables += parser(0).parse(lineText)
- lineText = BufferedReader.readLine()
+ lineText = bufferedReader.readLine()
}
metadata = new ScriptMetaData(variables.toArray)
metadata
@@ -66,7 +63,7 @@
def init(): Unit = {
inputStreamReader = new InputStreamReader(inputStream)
- BufferedReader = new BufferedReader(inputStreamReader)
+ bufferedReader = new BufferedReader(inputStreamReader)
}
@scala.throws[IOException]
@@ -82,25 +79,26 @@
override def available: Long = if (inputStream != null) inputStream.available() else 0L
override def close(): Unit = {
- if (BufferedReader != null) BufferedReader.close()
- if (inputStreamReader != null) inputStreamReader.close()
- if (inputStream != null) inputStream.close()
+ IOUtils.closeQuietly(bufferedReader)
+ IOUtils.closeQuietly(inputStreamReader)
+ IOUtils.closeQuietly(inputStream)
}
/**
* Determine if the read line is metadata(判断读的行是否是metadata)
+ *
* @param line
* @return
*/
- def isMetadata(line: String, prefix: String,prefixConf:String): Boolean = {
+ def isMetadata(line: String, prefix: String, prefixConf: String): Boolean = {
val regex = ("\\s*" + prefix + "\\s*(.+)\\s*" + "=" + "\\s*(.+)\\s*").r
line match {
- case regex(_,_) => true
+ case regex(_, _) => true
case _ => {
val split: Array[String] = line.split("=")
- if(split.size !=2) return false
- if (split(0).split(" ").filter(_!="").size != 4) return false
- if(!split(0).split(" ").filter(_!="")(0).equals(prefixConf)) return false
+ if (split.size != 2) return false
+ if (split(0).split(" ").filter(_ != "").size != 4) return false
+ if (!split(0).split(" ").filter(_ != "")(0).equals(prefixConf)) return false
true
}
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/writer/StorageScriptFsWriter.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/writer/StorageScriptFsWriter.scala
index 8f223eb..d506944 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/writer/StorageScriptFsWriter.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/script/writer/StorageScriptFsWriter.scala
@@ -16,43 +16,57 @@
package com.webank.wedatasphere.linkis.storage.script.writer
-import java.io.{IOException, OutputStream}
+import java.io.{ByteArrayInputStream, IOException, InputStream, OutputStream}
import java.util
import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
-import com.webank.wedatasphere.linkis.storage.script.{Compaction, ScriptFsWriter, ScriptMetaData, ScriptRecord}
-import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import com.webank.wedatasphere.linkis.storage.LineRecord
+import com.webank.wedatasphere.linkis.storage.script.{Compaction, ScriptFsWriter, ScriptMetaData}
+import com.webank.wedatasphere.linkis.storage.utils.{StorageConfiguration, StorageUtils}
import org.apache.commons.io.IOUtils
/**
* Created by johnnwang on 2018/10/23.
*/
-class StorageScriptFsWriter(pathP: FsPath, charsetP: String, outputStreamP: OutputStream) extends ScriptFsWriter {
- override val path: FsPath = pathP
- override val charset: String = charsetP
- override val outputStream: OutputStream = outputStreamP
+class StorageScriptFsWriter(val path: FsPath, val charset: String, outputStream: OutputStream = null) extends ScriptFsWriter {
+
+ private val stringBuilder = new StringBuilder
@scala.throws[IOException]
override def addMetaData(metaData: MetaData): Unit = {
val compactions = Compaction.listCompactions().filter(p => p.belongTo(StorageUtils.pathToSuffix(path.getPath)))
- val metadataLineJ = new util.ArrayList[String]()
+ val metadataLine = new util.ArrayList[String]()
if (compactions.length > 0) {
- metaData.asInstanceOf[ScriptMetaData].getMetaData.map(compactions(0).compact).foreach(metadataLineJ.add)
- IOUtils.writeLines(metadataLineJ, "\n", outputStream, charset)
+ metaData.asInstanceOf[ScriptMetaData].getMetaData.map(compactions(0).compact).foreach(metadataLine.add)
+ if (outputStream != null) {
+ IOUtils.writeLines(metadataLine, "\n", outputStream, charset)
+ } else {
+ import scala.collection.JavaConversions._
+ metadataLine.foreach(m => stringBuilder.append(s"$m\n"))
+ }
}
}
@scala.throws[IOException]
override def addRecord(record: Record): Unit = {
- val scriptRecord = record.asInstanceOf[ScriptRecord]
- IOUtils.write(scriptRecord.getLine, outputStream, charset)
+ //转成LineRecord而不是TableRecord是为了兼容非Table类型的结果集写到本类中
+ val scriptRecord = record.asInstanceOf[LineRecord]
+ if (outputStream != null) {
+ IOUtils.write(scriptRecord.getLine, outputStream, charset)
+ } else {
+ stringBuilder.append(scriptRecord.getLine)
+ }
}
override def close(): Unit = {
- StorageUtils.close(outputStream)
+ IOUtils.closeQuietly(outputStream)
}
override def flush(): Unit = if (outputStream != null) outputStream.flush()
+ def getInputStream(): InputStream = {
+ new ByteArrayInputStream(stringBuilder.toString().getBytes(StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue))
+ }
+
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/AbstractFileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/AbstractFileSource.scala
new file mode 100644
index 0000000..ee34e24
--- /dev/null
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/AbstractFileSource.scala
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.storage.source
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.io.{FsReader, FsWriter, MetaData, Record}
+import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord}
+import com.webank.wedatasphere.linkis.storage.script.{ScriptMetaData, VariableParser}
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
+import org.apache.commons.io.IOUtils
+import org.apache.commons.math3.util.Pair
+
+import scala.collection.JavaConversions._
+
+/**
+ * Created by johnnwang on 2020/1/15.
+ */
+abstract class AbstractFileSource extends FileSource {
+
+ protected var start: Int = 0
+
+ protected var end: Int = -1
+
+ protected var count: Int = 0
+
+ protected var totalLine = 0
+
+ protected var shuffler: Record => Record = r => r
+
+ protected var pageTrigger: Boolean = false
+
+ protected var params: util.Map[String, String] = new util.HashMap[String, String]
+
+ protected var fsReader: FsReader[_ <: MetaData, _ <: Record] = _
+
+ protected var `type`: String = "script/text"
+
+ def setType(`type`: String): AbstractFileSource = {
+ params += "type" -> `type`
+ this.`type` = `type`
+ this
+ }
+
+ override def shuffle(s: Record => Record): AbstractFileSource = {
+ this.shuffler = s
+ this
+ }
+
+ override def page(page: Int, pageSize: Int): AbstractFileSource = {
+ if (pageTrigger) return this
+ start = (page - 1) * pageSize
+ end = pageSize * page - 1
+ pageTrigger = true
+ this
+ }
+
+ override def addParams(params: util.Map[String, String]): AbstractFileSource = {
+ this.params.putAll(params)
+ this
+ }
+
+ override def addParams(key: String, value: String): FileSource = {
+ this.params += key -> value
+ this
+ }
+
+ override def getParams(): util.Map[String, String] = this.params
+
+ def setFsReader[K <: MetaData, V <: Record](fsReader: FsReader[K, V]): AbstractFileSource = {
+ this.fsReader = fsReader
+ this
+ }
+
+ override def write[K <: MetaData, V <: Record](fsWriter: FsWriter[K, V]): Unit = {
+ `while`(fsWriter.addMetaData, fsWriter.addRecord)
+ }
+
+ def `while`[M](m: MetaData => M, r: Record => Unit): M = {
+ val metaData = fsReader.getMetaData
+ val t = m(metaData)
+ while (fsReader.hasNext && ifContinueRead) {
+ if (ifStartRead) {
+ r(shuffler(fsReader.getRecord))
+ totalLine += 1
+ }
+ count += 1
+ }
+ t
+ }
+
+ override def close(): Unit = IOUtils.closeQuietly(this.fsReader)
+
+ def collectRecord(record: Record): Array[String] = {
+ record match {
+ case t: TableRecord => t.row.map(_.toString)
+ case l: LineRecord => Array(l.getLine)
+ }
+ }
+
+ override def collect(): Pair[Object, util.ArrayList[Array[String]]] = {
+ val record = new util.ArrayList[Array[String]]
+ val metaData = `while`(collectMetaData, r => record.add(collectRecord(r)))
+ this.params += "totalLine" -> totalLine.toString
+ new Pair(metaData, record)
+ }
+
+ //如果不分页,则一直读,如果分页,则 count需要小于count
+ def ifContinueRead: Boolean = !pageTrigger || count <= end
+
+ def ifStartRead: Boolean = !pageTrigger || count >= start
+
+ def collectMetaData(metaData: MetaData): Object = {
+ //script/text ,tableResultset,lineResultSet
+ metaData match {
+ case s: ScriptMetaData => VariableParser.getMap(s.getMetaData)
+ case l: LineMetaData => l.getMetaData
+ case t: TableMetaData => t.columns.map(_.toString)
+ }
+ }
+
+}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/FileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/FileSource.scala
new file mode 100644
index 0000000..05ecbea
--- /dev/null
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/FileSource.scala
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.storage.source
+
+import java.io.{Closeable, InputStream}
+import java.util
+
+import com.webank.wedatasphere.linkis.common.io._
+import com.webank.wedatasphere.linkis.storage.exception.StorageErrorException
+import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
+import com.webank.wedatasphere.linkis.storage.script.ScriptFsReader
+import com.webank.wedatasphere.linkis.storage.utils.StorageConfiguration
+import org.apache.commons.math3.util.Pair
+
+/**
+ * Created by johnnwang on 2020/1/15.
+ */
+trait FileSource extends Closeable {
+
+ def shuffle(s: Record => Record): FileSource
+
+ def page(page: Int, pageSize: Int): FileSource
+
+ def collect(): Pair[Object, util.ArrayList[Array[String]]]
+
+ def write[K <: MetaData, V <: Record](fsWriter: FsWriter[K, V]): Unit
+
+ def addParams(params: util.Map[String, String]): FileSource
+
+ def addParams(key: String, value: String): FileSource
+
+ def getParams(): util.Map[String, String]
+
+}
+
+object FileSource {
+
+ private val fileType = Array("dolphin", "sql", "scala", "py", "hql", "python", "out", "log", "text", "sh", "jdbc", "mlsql")
+
+ private val suffixPredicate = (path: String, suffix: String) => path.endsWith(s".$suffix")
+
+ def isResultSet(path: String): Boolean = {
+ suffixPredicate(path, fileType.head)
+ }
+
+ def isTableResultSet(fileSource: FileSource): Boolean = {
+ ResultSetFactory.TABLE_TYPE.equals(fileSource.getParams().get("type"))
+ }
+
+ def create(fsPath: FsPath, fs: Fs): FileSource = {
+ create(fsPath, fs.read(fsPath))
+ }
+
+ def create(fsPath: FsPath, is: InputStream): FileSource = {
+ canRead(fsPath.getPath)
+ if (isResultSet(fsPath.getPath)) {
+ val resultset = ResultSetFactory.getInstance.getResultSetByPath(fsPath)
+ val resultsetReader = ResultSetReader.getResultSetReader(resultset, is)
+ new ResultsetFileSource().setFsReader(resultsetReader).setType(resultset.resultSetType())
+ } else {
+ val scriptFsReader = ScriptFsReader.getScriptFsReader(fsPath, StorageConfiguration.STORAGE_RS_FILE_TYPE.getValue, is)
+ new TextFileSource().setFsReader(scriptFsReader)
+ }
+ }
+
+ private def canRead(path: String) = {
+ if (!fileType.exists(suffixPredicate(path, _))) throw new StorageErrorException(54001, "不支持打开的文件类型")
+ }
+
+}
+
+
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/ResultsetFileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/ResultsetFileSource.scala
new file mode 100644
index 0000000..4e237e9
--- /dev/null
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/ResultsetFileSource.scala
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.storage.source
+
+import com.webank.wedatasphere.linkis.storage.resultset.table.TableRecord
+import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+
+/**
+ * Created by johnnwang on 2020/1/15.
+ */
+class ResultsetFileSource extends AbstractFileSource {
+
+ shuffler = {
+ case t: TableRecord => new TableRecord(t.row.map {
+ case null => params.getOrDefault("nullValue", "NULL")
+ case value: Double => StorageUtils.doubleToString(value)
+ case r => r
+ })
+ case record => record
+ }
+
+}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/TextFileSource.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/TextFileSource.scala
new file mode 100644
index 0000000..0c37329
--- /dev/null
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/source/TextFileSource.scala
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.storage.source
+
+import java.util
+
+import com.webank.wedatasphere.linkis.storage.LineRecord
+import com.webank.wedatasphere.linkis.storage.script.ScriptRecord
+import org.apache.commons.math3.util.Pair
+
+/**
+ * Created by johnnwang on 2020/1/15.
+ */
+class TextFileSource extends AbstractFileSource {
+
+ shuffler = {
+ case s: ScriptRecord if "".equals(s.getLine) => new LineRecord("\n")
+ case record => record
+ }
+
+ override def collect(): Pair[Object, util.ArrayList[Array[String]]] = {
+ val collect: Pair[Object, util.ArrayList[Array[String]]] = super.collect()
+ if (!params.getOrDefault("ifMerge", "true").toBoolean) return collect
+ import scala.collection.JavaConversions._
+ val snd: util.ArrayList[Array[String]] = collect.getSecond
+ val str = new StringBuilder
+ for (i <- snd) {
+ i match {
+ case Array("\n") => str.append("\n")
+ case Array(y) => str.append(y).append("\n")
+ }
+ }
+ snd.clear()
+ snd.add(Array(str.toString()))
+ collect
+ }
+
+}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageConfiguration.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageConfiguration.scala
index 0729d1d..c49ae2a 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageConfiguration.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageConfiguration.scala
@@ -57,4 +57,6 @@
val IO_INIT_RETRY_LIMIT = CommonVars("wds.linkis.storage.io.init.retry.limit", 10)
val STORAGE_HDFS_GROUP = CommonVars("wds.linkis.storage.fileSystem.hdfs.group", "hadoop")
+
+ val DOUBLE_FRACTION_LEN = CommonVars[Int]("wds.linkis.double.fraction.length", 30)
}
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala
index 3878405..c4b2f15 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageHelper.scala
@@ -16,10 +16,9 @@
package com.webank.wedatasphere.linkis.storage.utils
-import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.storage.FSFactory
-import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord}
+import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
/**
* Created by johnnwang on 2019/3/7.
diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala
index b3ae098..5a57a05 100644
--- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala
+++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala
@@ -18,13 +18,14 @@
import java.io.{Closeable, File, InputStream, OutputStream}
import java.lang.reflect.Method
+import java.text.NumberFormat
import com.webank.wedatasphere.linkis.common.conf.Configuration
import com.webank.wedatasphere.linkis.common.io.{Fs, FsPath}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.storage.exception.StorageFatalException
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader, ResultSetWriter}
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
import org.apache.commons.lang.StringUtils
import scala.collection.mutable
@@ -41,6 +42,13 @@
val FILE_SCHEMA = "file://"
val HDFS_SCHEMA = "hdfs://"
+ private val nf = NumberFormat.getInstance()
+ nf.setGroupingUsed(false)
+ nf.setMaximumFractionDigits(StorageConfiguration.DOUBLE_FRACTION_LEN.getValue)
+
+ def doubleToString(value:Double): String ={
+ nf.format(value)
+ }
def loadClass[T](classStr: String, op: T => String): Map[String, T] = {
val _classes = classStr.split(",")
diff --git a/ujes/client/pom.xml b/ujes/client/pom.xml
index b1f1b67..a293cfd 100644
--- a/ujes/client/pom.xml
+++ b/ujes/client/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-client</artifactId>
diff --git a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala
index 5c72180..61c84b1 100644
--- a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala
+++ b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala
@@ -23,6 +23,7 @@
import java.util
import java.util.Date
+import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult
import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
@@ -48,9 +49,16 @@
val updatedTime = task.get("updatedTime").asInstanceOf[Long]
task.remove("createdTime")
task.remove("updatedTime")
- BeanUtils.populate(requestPersistTask, task)
+ task.remove("engineStartTime")
+ Utils.tryCatch{
+ BeanUtils.populate(requestPersistTask, task.asInstanceOf[util.Map[String, _]])
+ }{
+ case e:Exception => error("copy failed", e)
+ }
+ requestPersistTask.setStatus(task.get("status").asInstanceOf[String])
requestPersistTask.setCreatedTime(new Date(createdTime))
requestPersistTask.setUpdatedTime(new Date(updatedTime))
+ requestPersistTask.setEngineStartTime(new Date(updatedTime))
}
def getTask = task
diff --git a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala
index 4b9b4d7..b6988eb 100644
--- a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala
+++ b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala
@@ -24,7 +24,6 @@
import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
-
import org.json4s._
import org.json4s.jackson.Serialization._
diff --git a/ujes/definedEngines/hive/engine/pom.xml b/ujes/definedEngines/hive/engine/pom.xml
index 68260f2..54357a8 100644
--- a/ujes/definedEngines/hive/engine/pom.xml
+++ b/ujes/definedEngines/hive/engine/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala
index a7e57f3..61455f4 100644
--- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala
+++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala
@@ -18,7 +18,7 @@
import com.webank.wedatasphere.linkis.engine.execute.hook._
import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SQLCodeParser}
-import com.webank.wedatasphere.linkis.engine.hive.hook.{HiveAddJarsEngineHook, UseDatabaseEngineHook}
+import com.webank.wedatasphere.linkis.engine.hive.hook.HiveAddJarsEngineHook
import org.slf4j.LoggerFactory
import org.springframework.context.annotation.{Bean, Configuration}
diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala
index b3aba20..0faf90f 100644
--- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala
+++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala
@@ -19,14 +19,13 @@
import java.io.File
import java.nio.file.Paths
+import com.webank.wedatasphere.linkis.common.conf.{Configuration => CommonConfiguration}
import com.webank.wedatasphere.linkis.engine.hive.exception.HadoopConfSetFailedException
-
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.hive.conf
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.ql.Driver
-import com.webank.wedatasphere.linkis.common.conf.{Configuration => CommonConfiguration}
/**
* created by cooperyang on 2018/11/22
diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala
index c0ace13..b12cc4f 100644
--- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala
+++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala
@@ -36,10 +36,10 @@
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.metastore.api.{FieldSchema, Schema}
+import org.apache.hadoop.hive.ql.Driver
import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper
import org.apache.hadoop.hive.ql.processors.{CommandProcessor, CommandProcessorFactory, CommandProcessorResponse}
import org.apache.hadoop.hive.ql.session.SessionState
-import org.apache.hadoop.hive.ql.Driver
import org.apache.hadoop.mapred.RunningJob
import org.apache.hadoop.security.UserGroupInformation
import org.slf4j.LoggerFactory
diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala
index 5bade51..728925a 100644
--- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala
+++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala
@@ -22,8 +22,6 @@
import org.apache.commons.lang.StringUtils
import org.slf4j.{Logger, LoggerFactory}
-import scala.collection.mutable.ArrayBuffer
-
/**
* created by cooperyang on 2019/4/22
* Description:
diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala
index 047f2c0..cf1188b 100644
--- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala
+++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.engine.hive.progress
-import java.util
import java.util.concurrent.LinkedBlockingQueue
import com.webank.wedatasphere.linkis.engine.hive.log.HiveProgress
diff --git a/ujes/definedEngines/hive/enginemanager/pom.xml b/ujes/definedEngines/hive/enginemanager/pom.xml
index 84c66a4..949d601 100644
--- a/ujes/definedEngines/hive/enginemanager/pom.xml
+++ b/ujes/definedEngines/hive/enginemanager/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala b/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala
index 1a8a358..897c9dc 100644
--- a/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala
+++ b/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala
@@ -19,15 +19,14 @@
import java.nio.file.Paths
import com.webank.wedatasphere.linkis.common.conf.Configuration
-import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineCreator, EngineResource}
-import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration.{DEFAULT_JAVA_OPTS, ENGINE_CLIENT_MEMORY, JAVA_HOME, engineGCLogPath}
+import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration.{DEFAULT_JAVA_OPTS, JAVA_HOME, engineGCLogPath}
import com.webank.wedatasphere.linkis.enginemanager.hive.conf.HiveEngineConfiguration
import com.webank.wedatasphere.linkis.enginemanager.impl.UserEngineResource
import com.webank.wedatasphere.linkis.enginemanager.process.JavaProcessEngineBuilder
+import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineCreator, EngineResource}
import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine
import org.apache.commons.lang.StringUtils
import org.slf4j.LoggerFactory
-import org.springframework.stereotype.Component
import scala.collection.mutable.ArrayBuffer
diff --git a/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh b/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh
index 56cab6d..80cc775 100644
--- a/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh
+++ b/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/ujes/definedEngines/hive/entrance/pom.xml b/ujes/definedEngines/hive/entrance/pom.xml
index b39b7c1..bb771d4 100644
--- a/ujes/definedEngines/hive/entrance/pom.xml
+++ b/ujes/definedEngines/hive/entrance/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh b/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh
index 56cab6d..80cc775 100644
--- a/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh
+++ b/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/ujes/definedEngines/jdbc/entrance/pom.xml b/ujes/definedEngines/jdbc/entrance/pom.xml
index 582c39a..bba9386 100644
--- a/ujes/definedEngines/jdbc/entrance/pom.xml
+++ b/ujes/definedEngines/jdbc/entrance/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -77,7 +77,7 @@
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.9.3</version>
+ <version>0.9.2</version>
<type>pom</type>
</dependency>
</dependencies>
diff --git a/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java b/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java
index 793f4c5..c3e3772 100644
--- a/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java
+++ b/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java
@@ -18,6 +18,8 @@
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.dbcp.BasicDataSourceFactory;
import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.sql.DataSource;
import java.sql.Connection;
@@ -27,8 +29,6 @@
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class ConnectionManager {
diff --git a/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala b/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala
index c42ba79..50ccfe7 100644
--- a/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala
+++ b/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala
@@ -23,7 +23,6 @@
import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
-import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.context.annotation.Configuration
@Configuration
diff --git a/ujes/definedEngines/mlsql/entrance/pom.xml b/ujes/definedEngines/mlsql/entrance/pom.xml
index 62f924d..31d60e7 100644
--- a/ujes/definedEngines/mlsql/entrance/pom.xml
+++ b/ujes/definedEngines/mlsql/entrance/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/pipeline/engine/pom.xml b/ujes/definedEngines/pipeline/engine/pom.xml
index 956cac8..67c2155 100644
--- a/ujes/definedEngines/pipeline/engine/pom.xml
+++ b/ujes/definedEngines/pipeline/engine/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/ujes/definedEngines/pipeline/engine/src/main/java/com/webank/wedatasphere/linkis/engine/pipeline/domain/PipeEntity.java b/ujes/definedEngines/pipeline/engine/src/main/java/com/webank/wedatasphere/linkis/engine/pipeline/domain/PipeEntity.java
deleted file mode 100644
index 0091fe4..0000000
--- a/ujes/definedEngines/pipeline/engine/src/main/java/com/webank/wedatasphere/linkis/engine/pipeline/domain/PipeEntity.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.engine.pipeline.domain;
-
-import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException;
-import com.webank.wedatasphere.linkis.engine.pipeline.parser.IEParser;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Created by johnnwang on 2018/11/14.
- */
-public class PipeEntity {
-
- private final Logger LOGGER = LoggerFactory.getLogger(getClass());
-
- public static final String SET_REGEX = "\\s*set\\s+([a-z]{2,12})\\s*=\\s*(\\S+)\\s?";
- public static final String STORAGE_REGEX = "(?i)\\s*from\\s+(\\S+)\\s+to\\s+(\\S+)\\s?";
- static final Pattern SET_PATTERN = Pattern.compile(SET_REGEX);
- static final Pattern STORAGE_PATTERN = Pattern.compile(STORAGE_REGEX);
- private String source;
- private String dest;
- private String type;
- private Map<String, String> paramMap = new HashMap<String, String>();
-
- public PipeEntity(String script) throws PipeLineErrorException {
-
-
- /**
- * script样式：
- * %pipeLine.toWorkspace
- * set key=value;
- * set key=value;
- * set key=value;
- * set key=value;
- * from source to dest;
- */
- String kind = IEParser.getKind(script);
- if ("pipeLine.toWorkspace".equals(kind)) {
- this.type = "toWorkspace";
- } else if ("pipeLine.toHDFS".equals(kind)) {
- this.type = "toHDFS";
- } else if ("pipeLine.toExcel".equals(kind)) {
- this.type = "toExcel";
- } else if ("pipeLine.toCSV".equals(kind)) {
- this.type = "toCSV";
- } else if ("pipeLine.toTxt".equals(kind)) {
- this.type = "toTxt";
- } else {
- // TODO: 2018/11/16 错误码待定
- LOGGER.error("not support(不支持) " + kind + " Import and export type(的导入导出类型)");
- throw new PipeLineErrorException(70001, "不支持" + kind + "的导入导出类型");
- }
- String code = IEParser.getFormatCode(script);
- if (StringUtils.isEmpty(code)) {
- LOGGER.error("code is Empty!");
- throw new PipeLineErrorException(70002, "The execution code is empty!(执行代码为空!)");
- }
- String splitRegex = ";";
- if (code.indexOf(";") < 0) {
- splitRegex = "\n";
- }
- String[] codes = code.split(splitRegex);
- for (String _code : codes) {
- Matcher matcher = SET_PATTERN.matcher(_code);
- boolean find = matcher.find();
- if (find) {
- String key = matcher.group(1);
- String value = matcher.group(2);
- paramMap.put(key, value);
- //else ignore it.
- continue;
- }
- matcher = STORAGE_PATTERN.matcher(_code);
- find = matcher.find();
- if (find) {
- source = matcher.group(1);
- dest = matcher.group(2);
- } //else ignore it.
- }
- if (StringUtils.isEmpty(source) || StringUtils.isEmpty(dest)) {
- LOGGER.error("Import or export address is empty(导入或导出地址为空)");
- throw new PipeLineErrorException(70003, "Import or export address is empty（导入或导出地址为空）");
- }
- }
-
- @Override
- public String toString() {
- return "PipeEntity{" +
- "source='" + source + '\'' +
- ", dest='" + dest + '\'' +
- ", type='" + type + '\'' +
- ", paramMap=" + paramMap +
- '}';
- }
-
- public String getSource() {
- return source;
- }
-
- public void setSource(String source) {
- this.source = source;
- }
-
- public String getDest() {
- return dest;
- }
-
- public void setDest(String dest) {
- this.dest = dest;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public Map<String, String> getParamMap() {
- return paramMap;
- }
-
- public void setParamMap(Map<String, String> paramMap) {
- this.paramMap = paramMap;
- }
-}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/OutputStreamCache.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/OutputStreamCache.scala
new file mode 100644
index 0000000..c07a472
--- /dev/null
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/OutputStreamCache.scala
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.engine.pipeline
+
+import java.io.OutputStream
+
+/**
+ * created by cooperyang on 2019/12/9
+ * Description:
+ */
+object OutputStreamCache {
+ final val osCache = new java.util.HashMap[String, OutputStream]()
+}
\ No newline at end of file
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeEngineJob.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeEngineJob.scala
new file mode 100644
index 0000000..76d51cd
--- /dev/null
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeEngineJob.scala
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.engine.pipeline
+
+import com.webank.wedatasphere.linkis.engine.execute.CommonEngineJob
+
+/**
+ * created by cooperyang on 2019/12/9
+ * Description:
+ */
+class PipeEngineJob extends CommonEngineJob {
+ override def kill(): Unit = {
+ info("begin to remove osCache:" + this.getId)
+ OutputStreamCache.osCache.get(this.getId).close() //因为SingleTaskInfoSupport，应该是没有并发问题的
+ super.kill()
+ }
+}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutor.scala
index a0026aa..9e1c434 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutor.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutor.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.engine.pipeline
-
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext}
import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
@@ -29,8 +28,8 @@
import com.webank.wedatasphere.linkis.server._
/**
- * Created by johnnwang on 2018/11/13.
- */
+ * Created by johnnwang on 2018/11/13.
+ */
class PipeLineEngineExecutor(options: JMap[String, String]) extends EngineExecutor(outputPrintLimit = 10, false) with SingleTaskInfoSupport with Logging {
override def getName: String = "pipeLineEngine"
@@ -38,7 +37,7 @@
private var index = 0
private var progressInfo: JobProgressInfo = _
- override def getActualUsedResources: Resource = new LoadInstanceResource(Runtime.getRuntime.totalMemory() - Runtime.getRuntime.freeMemory(), 2, 1)
+ override def getActualUsedResources: Resource = new LoadInstanceResource(Runtime.getRuntime.totalMemory() - Runtime.getRuntime.freeMemory(), 1, 1)
override def init(): Unit = {
info("init pipeLineEngine...")
@@ -64,9 +63,9 @@
code match {
case regex(sourcePath, destPath) => {
if (destPath.contains(".")) {
- PipeLineExecutorFactory.listPipeLineExecutor.find(f => "cp".equals(f.Kind)).get.execute(sourcePath, destPath)
+ PipeLineExecutorFactory.listPipeLineExecutor.find(f => "cp".equals(f.Kind)).get.execute(sourcePath, destPath, engineExecutorContext)
} else {
- PipeLineExecutorFactory.listPipeLineExecutor.find(f => newOptions.get("pipeline.output.mold").equalsIgnoreCase(f.Kind)).map(_.execute(sourcePath, destPath)).get
+ PipeLineExecutorFactory.listPipeLineExecutor.find(f => newOptions.get("pipeline.output.mold").equalsIgnoreCase(f.Kind)).map(_.execute(sourcePath, destPath, engineExecutorContext)).get
}
}
case _ => throw new PipeLineErrorException(70007, "")
@@ -75,6 +74,8 @@
case e: Exception => failedTasks = 1; succeedTasks = 0; throw e
}
finally {
+ info("begin to remove osCache:" + engineExecutorContext.getJobId.get)
+ OutputStreamCache.osCache.remove(engineExecutorContext.getJobId.get)
progressInfo = JobProgressInfo(getName + "_" + index, 1, 0, failedTasks, succeedTasks)
}
}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutorFactory.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutorFactory.scala
index 202a6df..d80df42 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutorFactory.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeLineEngineExecutorFactory.scala
@@ -17,14 +17,12 @@
package com.webank.wedatasphere.linkis.engine.pipeline
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorFactory}
-import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
-import com.webank.wedatasphere.linkis.engine.pipeline.executor.{CSVExecutor, CopyExecutor, ExcelExecutor}
import com.webank.wedatasphere.linkis.server.JMap
import org.springframework.stereotype.Component
/**
- * Created by johnnwang on 2018/11/13.
- */
+ * Created by johnnwang on 2018/11/13.
+ */
@Component("engineExecutorFactory")
class PipeLineEngineExecutorFactory extends EngineExecutorFactory {
override def createExecutor(options: JMap[String, String]): EngineExecutor = new PipeLineEngineExecutor(options)
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeTest.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeTest.scala
deleted file mode 100644
index a354a4a..0000000
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeTest.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.engine.pipeline
-
-import com.webank.wedatasphere.linkis.engine.pipeline.domain.PipeEntity
-import com.webank.wedatasphere.linkis.engine.pipeline.util.PipeLineUtil
-
-/**
- * Created by johnnwang on 2018/11/14.
- */
-object PipeTest {
- def main(args: Array[String]): Unit = {
- val out01 = "from file:///appcom/bdap/johnnwang/ to hdfs:///tmp/bdp-ide/johnnwang/test1/new_py_180910175033.python"
- val regex = ("(?i)\\s*from\\s+(\\S+)\\s+to\\s+(\\S+)\\s?").r
- out01 match {
- case regex(source,desct) => print(source.contains("."))
- }
-
- }
-
-}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipelineEngineParser.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipelineEngineParser.scala
new file mode 100644
index 0000000..fb6db27
--- /dev/null
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipelineEngineParser.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.engine.pipeline
+
+import com.webank.wedatasphere.linkis.engine.EngineParser
+import com.webank.wedatasphere.linkis.protocol.engine.RequestTask
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+import org.springframework.stereotype.Component
+
+/**
+ * created by cooperyang on 2019/12/9
+ * Description:
+ */
+@Component("engineParser")
+class PipelineEngineParser extends EngineParser {
+ override def parseToJob(request: RequestTask): Job = {
+ val job = new PipeEngineJob
+ job.setRequestTask(request)
+ job
+ }
+}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/constant/PipeLineConstant.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/constant/PipeLineConstant.scala
new file mode 100644
index 0000000..80cd703
--- /dev/null
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/constant/PipeLineConstant.scala
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.pipeline.constant
+
+/**
+ * Created by johnnwang on 2018/11/16.
+ */
+object PipeLineConstant {
+ val DEFAULTC_HARSET = "utf-8"
+ val DEFAULT_SHEETNAME = "result"
+ val DEFAULT_DATEFORMATE = "yyyy-MM-dd HH:mm:ss"
+ val PIPELINE_OUTPUT_ISOVERWRITE = "pipeline.output.isoverwtite"
+ val PIPELINE_OUTPUT_SHUFFLE_NULL_TYPE = "pipeline.output.shuffle.null.type"
+ val PIPELINE_OUTPUT_CHARSET = "pipeline.output.charset"
+ val PIPELINE_FIELD_SPLIT = "pipeline.field.split"
+ val BLANK = "BLANK"
+}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConversions.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConversions.scala
new file mode 100644
index 0000000..dddc624
--- /dev/null
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConversions.scala
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.pipeline.conversions
+
+import java.io.Closeable
+
+import com.webank.wedatasphere.linkis.common.io.Fs
+
+/**
+ * Created by patinousward on 2020/2/4.
+ */
+object FsConversions {
+
+ implicit def fsToFsClosable(fs: Fs): Closeable = {
+ new FsClosable(fs)
+ }
+}
+
+class FsClosable(fs: Fs) extends Closeable {
+ override def close(): Unit = {
+ fs.close()
+ }
+}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala
new file mode 100644
index 0000000..a936d7e
--- /dev/null
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.pipeline.conversions
+
+import java.io.Closeable
+
+import com.webank.wedatasphere.linkis.common.io.Fs
+
+/**
+ * Created by johnnwang on 2020/2/4.
+ */
+object FsConvertions {
+
+ implicit def fsToFsClosable(fs: Fs): Closeable = {
+ new FsClosable(fs)
+ }
+}
+
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/exception/PipeLineErrorException.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/exception/PipeLineErrorException.scala
index 2f3a8ec..9e4b409 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/exception/PipeLineErrorException.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/exception/PipeLineErrorException.scala
@@ -19,8 +19,8 @@
import com.webank.wedatasphere.linkis.common.exception.ErrorException
/**
- * Created by johnnwang on 2018/11/16.
- */
+ * Created by johnnwang on 2018/11/16.
+ */
class PipeLineErrorException(errCode: Int, desc: String) extends ErrorException(errCode, desc) {
}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala
index fd03498..18aaad5 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala
@@ -15,64 +15,61 @@
*/
package com.webank.wedatasphere.linkis.engine.pipeline.executor
-import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream, OutputStream}
+
+import java.io.OutputStream
import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache
+import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._
+import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._
import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
-import com.webank.wedatasphere.linkis.engine.pipeline.util.{PipeLineConstants, PipeLineUtils}
-import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
import com.webank.wedatasphere.linkis.server._
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.csv.CSVFsWriter
-import com.webank.wedatasphere.linkis.storage.excel.ExcelFsWriter
-import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
+import com.webank.wedatasphere.linkis.storage.source.FileSource
import org.apache.commons.io.IOUtils
-import org.springframework.stereotype.Component
/**
- * Created by johnnwang on 2019/1/30.
- */
+ * Created by johnnwang on 2019/1/30.
+ */
-class CSVExecutor extends PipeLineExecutor{
+class CSVExecutor extends PipeLineExecutor {
- override def execute(sourcePath: String, destPath: String): ExecuteResponse = {
- //val sourcePath = pipeEntity.getSource
- if (!PipeLineUtils.isDolphin(sourcePath)) throw new PipeLineErrorException(70005, "Not a result set file（不是结果集文件）")
+ override def execute(sourcePath: String, destPath: String, engineExecutorContext: EngineExecutorContext): ExecuteResponse = {
+ if (!FileSource.isResultSet(sourcePath)) {
+ throw new PipeLineErrorException(70005, "Not a result set file（不是结果集文件）")
+ }
val sourceFsPath = new FsPath(sourcePath)
- //val destPath = pipeEntity.getDest
- val destFsPath = new FsPath(destPath+"."+Kind)
+ val destFsPath = new FsPath(s"$destPath.$Kind")
val sourceFs = FSFactory.getFs(sourceFsPath)
sourceFs.init(null)
val destFs = FSFactory.getFs(destFsPath)
destFs.init(null)
- val resultset = ResultSetFactory.getInstance.getResultSetByPath(sourceFsPath)
- val reader = ResultSetReader.getResultSetReader(resultset, sourceFs.read(sourceFsPath))
- val metadata = reader.getMetaData
- if (!PipeLineUtils.isTableResultset(metadata)) throw new PipeLineErrorException(70005, "Only the result set of the table type can be converted to csv（只有table类型的结果集才能转为csv）");
- val cSVFsWriter = CSVFsWriter.getCSVFSWriter(options.get("pipeline.output.charset"),options.get("pipeline.field.split"))
- cSVFsWriter.addMetaData(metadata)
- while (reader.hasNext){
- cSVFsWriter.addRecord(reader.getRecord)
+ val fileSource = FileSource.create(sourceFsPath, sourceFs)
+ if (!FileSource.isTableResultSet(fileSource)) {
+ throw new PipeLineErrorException(70005, "只有table类型的结果集才能转为csv")
}
- val inputStream = cSVFsWriter.getCSVStream
- val outputStream:OutputStream = destFs.write(destFsPath,options.get("pipeline.output.isoverwtite").toBoolean)
- // TODO: a series of close(一系列的close)
- IOUtils.copy(inputStream,outputStream)
- IOUtils.closeQuietly(outputStream)
- IOUtils.closeQuietly(inputStream)
- if (cSVFsWriter != null) cSVFsWriter.close()
- if (reader != null) reader.close()
- if(sourceFs != null) sourceFs.close()
- if(destFs != null) destFs.close()
- cleanOptions
- SuccessExecuteResponse()
+ var nullValue = options.getOrDefault(PIPELINE_OUTPUT_SHUFFLE_NULL_TYPE, "NULL")
+ if (BLANK.equalsIgnoreCase(nullValue)) nullValue = ""
+ val outputStream: OutputStream = destFs.write(destFsPath, options.get(PIPELINE_OUTPUT_ISOVERWRITE).toBoolean)
+ OutputStreamCache.osCache += engineExecutorContext.getJobId.get -> outputStream
+ val cSVFsWriter = CSVFsWriter.getCSVFSWriter(options.get(PIPELINE_OUTPUT_CHARSET), options.get(PIPELINE_FIELD_SPLIT), outputStream)
+ fileSource.addParams("nullValue", nullValue).write(cSVFsWriter)
+ IOUtils.closeQuietly(cSVFsWriter)
+ IOUtils.closeQuietly(fileSource)
+ IOUtils.closeQuietly(sourceFs)
+ IOUtils.closeQuietly(destFs)
+ super.execute(sourcePath, destPath, engineExecutorContext)
}
override def Kind: String = "csv"
}
-object CSVExecutor{
+
+object CSVExecutor {
val csvExecutor = new CSVExecutor
- def getInstance:PipeLineExecutor = csvExecutor
+
+ def getInstance: PipeLineExecutor = csvExecutor
}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala
index 2f103b8..1a5afdb 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala
@@ -15,52 +15,53 @@
*/
package com.webank.wedatasphere.linkis.engine.pipeline.executor
+
import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
-import com.webank.wedatasphere.linkis.engine.pipeline.util.PipeLineConstants
-import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
+import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache
+import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._
+import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
import com.webank.wedatasphere.linkis.server._
import com.webank.wedatasphere.linkis.storage.FSFactory
import org.apache.commons.io.IOUtils
-import org.springframework.stereotype.Component
/**
- * Created by johnnwang on 2019/1/30.
- */
+ * Created by johnnwang on 2019/1/30.
+ */
-class CopyExecutor extends PipeLineExecutor{
- override def execute(sourcePath: String, destPath: String): ExecuteResponse = {
- //val sourcePath = pipeEntity.getSource
+class CopyExecutor extends PipeLineExecutor {
+ override def execute(sourcePath: String, destPath: String, engineExecutorContext: EngineExecutorContext): ExecuteResponse = {
val sourceFsPath = new FsPath(sourcePath)
- //val destPath = pipeEntity.getDest
val destFsPath = new FsPath(destPath)
val sourceFs = FSFactory.getFs(sourceFsPath)
sourceFs.init(null)
val destFs = FSFactory.getFs(destFsPath)
destFs.init(null)
val inputStream = sourceFs.read(sourceFsPath)
- var isOverWrite = options.get("pipeline.output.isoverwtite").toBoolean
- //The export table is currently only exported to the workspace, so other places are temporarily not modified.（导出表目前因为是只导出到工作空间，所以别的地方暂时不修改）
- if(!isOverWrite && !destFs.exists(destFsPath)){
+ var isOverWrite = options.get(PIPELINE_OUTPUT_ISOVERWRITE).toBoolean
+ //导出表目前因为是只导出到工作空间，所以别的地方暂时不修改
+ if (!isOverWrite && !destFs.exists(destFsPath)) {
isOverWrite = true
}
val outputStream = destFs.write(destFsPath, isOverWrite)
+ OutputStreamCache.osCache += engineExecutorContext.getJobId.get -> outputStream
IOUtils.copy(inputStream, outputStream)
- // TODO: a series of close（一系列的close）
- outputStream.close()
- inputStream.close()
- sourceFs.close()
- destFs.close()
- cleanOptions
- SuccessExecuteResponse()
+ IOUtils.closeQuietly(outputStream)
+ IOUtils.closeQuietly(inputStream)
+ IOUtils.closeQuietly(sourceFs)
+ IOUtils.closeQuietly(destFs)
+ super.execute(sourcePath, destPath, engineExecutorContext)
}
override def Kind: String = "cp"
}
-object CopyExecutor{
+object CopyExecutor {
val copyExecutor = new CopyExecutor
- def getInstance:PipeLineExecutor = copyExecutor
+
+ def getInstance: PipeLineExecutor = copyExecutor
}
+
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala
index 5894074..dde2f10 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala
@@ -15,62 +15,59 @@
*/
package com.webank.wedatasphere.linkis.engine.pipeline.executor
-import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream, OutputStream}
+
+import java.io.OutputStream
import com.webank.wedatasphere.linkis.common.io.FsPath
+import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache
+import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._
+import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._
import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
-import com.webank.wedatasphere.linkis.engine.pipeline.util.{PipeLineConstants, PipeLineUtils}
-import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
import com.webank.wedatasphere.linkis.storage.FSFactory
import com.webank.wedatasphere.linkis.storage.excel.ExcelFsWriter
-import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
+import com.webank.wedatasphere.linkis.storage.source.FileSource
import org.apache.commons.io.IOUtils
/**
- * Created by johnnwang on 2019/1/30.
- */
+ * Created by johnnwang on 2019/1/30.
+ */
class ExcelExecutor extends PipeLineExecutor {
- override def execute(sourcePath: String, destPath: String): ExecuteResponse = {
- //val sourcePath = pipeEntity.getSource
- if (!PipeLineUtils.isDolphin(sourcePath)) throw new PipeLineErrorException(70005, "Not a result set file（不是结果集文件）")
+ override def execute(sourcePath: String, destPath: String, engineExecutorContext: EngineExecutorContext): ExecuteResponse = {
+ if (!FileSource.isResultSet(sourcePath)) {
+ throw new PipeLineErrorException(70005, "不是结果集文件")
+ }
val sourceFsPath = new FsPath(sourcePath)
- // val destPath = pipeEntity.getDest
- val destFsPath = new FsPath(destPath+".xlsx")
+ val destFsPath = new FsPath(s"$destPath.xlsx")
val sourceFs = FSFactory.getFs(sourceFsPath)
sourceFs.init(null)
val destFs = FSFactory.getFs(destFsPath)
destFs.init(null)
- val resultset = ResultSetFactory.getInstance.getResultSetByPath(sourceFsPath)
- val reader = ResultSetReader.getResultSetReader(resultset, sourceFs.read(sourceFsPath))
- val metadata = reader.getMetaData
- if (!PipeLineUtils.isTableResultset(metadata)) throw new PipeLineErrorException(70005, "Only the result set of the table type can be converted to excel（只有table类型的结果集才能转为excel）")
- val excelFsWriter = ExcelFsWriter.getExcelFsWriter(PipeLineConstants.DEFAULTCHARSET, PipeLineConstants.DEFAULTSHEETNAME, PipeLineConstants.DEFAULTDATEFORMATE)
- excelFsWriter.addMetaData(metadata)
- while (reader.hasNext){
- excelFsWriter.addRecord(reader.getRecord)
+ val fileSource = FileSource.create(sourceFsPath, sourceFs)
+ if (!FileSource.isTableResultSet(fileSource)) {
+ throw new PipeLineErrorException(70005, "只有table类型的结果集才能转为excel")
}
- val os: ByteArrayOutputStream = new ByteArrayOutputStream()
- excelFsWriter.getWorkBook.write(os )
- val inputStream:InputStream = new ByteArrayInputStream(os.toByteArray)
- val outputStream:OutputStream = destFs.write(destFsPath,options.get("pipeline.output.isoverwtite").toBoolean)
- // TODO: a series of close(一系列的close)
- IOUtils.copy(inputStream,outputStream)
- IOUtils.closeQuietly(outputStream)
- IOUtils.closeQuietly(inputStream)
- IOUtils.closeQuietly(os)
- if (excelFsWriter != null) excelFsWriter.close()
- if (reader != null) reader.close()
- if(sourceFs != null) sourceFs.close()
- if(destFs != null) destFs.close()
- cleanOptions
- SuccessExecuteResponse()
+ var nullValue = options.getOrDefault(PIPELINE_OUTPUT_SHUFFLE_NULL_TYPE, "NULL")
+ if (BLANK.equalsIgnoreCase(nullValue)) nullValue = ""
+ val outputStream: OutputStream = destFs.write(destFsPath, options.get(PIPELINE_OUTPUT_ISOVERWRITE).toBoolean)
+ val excelFsWriter = ExcelFsWriter.getExcelFsWriter(DEFAULTC_HARSET, DEFAULT_SHEETNAME, DEFAULT_DATEFORMATE, outputStream)
+ import scala.collection.JavaConversions._
+ OutputStreamCache.osCache += engineExecutorContext.getJobId.get -> outputStream
+ fileSource.addParams("nullValue", nullValue).write(excelFsWriter)
+ IOUtils.closeQuietly(excelFsWriter)
+ IOUtils.closeQuietly(fileSource)
+ IOUtils.closeQuietly(sourceFs)
+ IOUtils.closeQuietly(destFs)
+ super.execute(sourcePath, destPath, engineExecutorContext)
}
- override def Kind: String = "excel"
+ override def Kind: String = "excel"
}
-object ExcelExecutor{
- val excelExecutor = new ExcelExecutor
- def getInstance:PipeLineExecutor = excelExecutor
+object ExcelExecutor {
+ val excelExecutor = new ExcelExecutor
+
+ def getInstance: PipeLineExecutor = excelExecutor
}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutor.scala
index ac08535..b570a11 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutor.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutor.scala
@@ -17,19 +17,24 @@
package com.webank.wedatasphere.linkis.engine.pipeline.executor
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
-import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
-import com.webank.wedatasphere.linkis.server._
+import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
/**
- * Created by johnnwang on 2019/1/30.
- */
+ * Created by johnnwang on 2019/1/30.
+ */
trait PipeLineExecutor {
var options: java.util.Map[String, String] = _
- def execute(sourcePath: String,destPath: String): ExecuteResponse
- def Kind:String
- def init(newOptions:java.util.Map[String, String]): Unit ={
+
+ def execute(sourcePath: String, destPath: String, engineExecutorContext: EngineExecutorContext): ExecuteResponse = {
+ cleanOptions
+ SuccessExecuteResponse()
+ }
+
+ def Kind: String
+
+ def init(newOptions: java.util.Map[String, String]): Unit = {
options = newOptions
}
- def cleanOptions= options = null
+ def cleanOptions = options = null
}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutorFactory.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutorFactory.scala
index ec9a432..debb93c 100644
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutorFactory.scala
+++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/PipeLineExecutorFactory.scala
@@ -16,11 +16,9 @@
package com.webank.wedatasphere.linkis.engine.pipeline.executor
-import org.springframework.beans.factory.annotation.Autowired
-
/**
- * Created by johnnwang on 2019/1/30.
- */
+ * Created by johnnwang on 2019/1/30.
+ */
object PipeLineExecutorFactory {
- def listPipeLineExecutor:Array[PipeLineExecutor] = Array(CopyExecutor.getInstance,CSVExecutor.getInstance,ExcelExecutor.getInstance)
+ def listPipeLineExecutor: Array[PipeLineExecutor] = Array(CopyExecutor.getInstance, CSVExecutor.getInstance, ExcelExecutor.getInstance)
}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/TxtExecutor.scala.bak b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/TxtExecutor.scala.bak
deleted file mode 100644
index 868638a..0000000
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/TxtExecutor.scala.bak
+++ /dev/null
@@ -1,48 +0,0 @@
-package com.webank.wedatasphere.linkis.engine.pipeline.executor
-import java.io.{ByteArrayInputStream, InputStream, OutputStream}
-
-import com.webank.wedatasphere.linkis.common.io.FsPath
-import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
-import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
-import com.webank.wedatasphere.linkis.engine.pipeline.util.{PipeLineConstants, PipeLineUtils}
-import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
-import com.webank.wedatasphere.linkis.server._
-import com.webank.wedatasphere.linkis.storage.{FSFactory, LineMetaData, LineRecord}
-import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
-import org.apache.commons.io.IOUtils
-
-/**
- * Created by johnnwang on 2019/1/30.
- */
-class TxtExecutor(options: JMap[String, String]) extends PipeLineExecutor{
- override def execute(code: String, engineExecutorContext: EngineExecutorContext, jobGroup: String): ExecuteResponse = {
- val sourcePath = pipeEntity.getSource
- if (!PipeLineUtils.isDolphin(sourcePath)) throw new PipeLineErrorException(70005, "不是结果集文件")
- val sourceFsPath = new FsPath(sourcePath)
- val destPath = pipeEntity.getDest
- val destFsPath = new FsPath(destPath)
- val sourceFs = FSFactory.getFs(sourceFsPath)
- sourceFs.init(null)
- val destFs = FSFactory.getFs(destFsPath)
- destFs.init(null)
- val resultset = ResultSetFactory.getInstance.getResultSetByPath(sourceFsPath)
- val reader = ResultSetReader.getResultSetReader(resultset, sourceFs.read(sourceFsPath))
- val metadata = reader.getMetaData
- if (!PipeLineUtils.isLineResultset(metadata)) throw new PipeLineErrorException(70006, "只有line类型的结果集才能转为txt");
- val content = new StringBuffer()
- content.append(metadata.asInstanceOf[LineMetaData].getMetaData)
- while(reader.hasNext){
- content.append(reader.getRecord.asInstanceOf[LineRecord].getLine)
- }
- val inputStream:InputStream = new ByteArrayInputStream(content.toString().getBytes(PipeLineConstants.DEFAULTCHARSET))
- val outputStream:OutputStream = destFs.write(destFsPath,PipeLineConstants.DEFAULTOVERWIRTE)
- // TODO: a series of close（一系列的close）
- IOUtils.copy(inputStream,outputStream)
- IOUtils.closeQuietly(outputStream)
- IOUtils.closeQuietly(inputStream)
- if (reader != null) reader.close()
- if(sourceFs != null) sourceFs.close()
- if(destFs != null) destFs.close()
- SuccessExecuteResponse()
- }
-}
diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/parser/IEParser.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/parser/IEParser.scala
deleted file mode 100644
index cc33c18..0000000
--- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/parser/IEParser.scala
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.engine.pipeline.parser
-
-import com.webank.wedatasphere.linkis.common.utils.Utils
-import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException
-import org.apache.commons.lang.StringUtils
-
-/**
- * Created by johnnwang on 2018/11/14.
- * import and export parser
- */
-object IEParser {
-
- val RESTART_CODE = "@restart"
-
- val APPLICATION_START_COMMAND = List("\\s*@restart\\s*",
- "\\s*[@|%][a-zA-Z]{1,12}\\s*",
- "^\\s*@set\\s*spark\\..+\\s*",
- "^\\s*#.+\\s*",
- "^\\s*//.+\\s*",
- "^\\s*--.+\\s*",
- "\\s*")
-
- def needToRestart(code: String): Boolean = code.startsWith(RESTART_CODE)
-
- private def getIndex(_code: String, start: Int): Int = {
- val index1 = _code.indexOf("\n", start)
- val index2 = _code.indexOf("\\n", start)
- if (index1 > -1 && index2 > -1) Math.min(index1, index2) else Math.max(index1, index2)
- }
-
- def getKindString(code: String): String = {
- val _code = StringUtils.strip(code)
-
- var start = 0
- if (_code.startsWith(RESTART_CODE)) {
- start = getIndex(_code, 0) + 1
- }
- var index = getIndex(_code, start)
- if (index == -1) index = _code.length
- StringUtils.strip(_code.substring(start, index))
- }
-
- def getKind(code: String): String = {
- val kindStr = getKindString(code)
- if (kindStr.matches("[%|@][a-zA-Z]{1,12}[\\.][a-zA-Z]{1,12}")) kindStr.substring(1) else throw new PipeLineErrorException(70004,"unknown kind")
- }
-
-
- /**
- * This method just removes @restart and language identifiers (such as %sql, %scala, etc.), that is, removes at most the first 2 rows.
- * 该方法只是去掉了@restart和语言标识符（如%sql、%scala等），即最多只去掉最前面2行
- *
- * @param code
- * @return
- */
- def getRealCode(code: String): String = {
- val _code = StringUtils.strip(code)
- val kindStr = getKindString(_code)
- if (kindStr.matches("[%|@][a-zA-Z]{1,12}"))
- StringUtils.strip(_code.substring(_code.indexOf(kindStr) + kindStr.length))
- else if (_code.startsWith(RESTART_CODE)) StringUtils.strip(_code.substring(RESTART_CODE.length))
- else _code
- }
-
- /**
- * This method removes all code-independent setting parameters and identifiers (including comments)
- * 该方法去掉一切与代码无关的设置参数和标识符（包括注释）
- *
- * @param code
- * @return
- */
- def getFormatCode(code: String): String = {
- val msg = new StringBuilder
- val restartRegex = "\\s*@restart\\s*".r
- val kindRegex = "\\s*[@|%][a-zA-Z]{1,12}\\s*".r
- val setRegex = "^\\s*@set\\s*.+\\s*".r
- val symbolRegex1 = "^\\s*#.+\\s*".r
- val symbolRegex2 = "^\\s*//.+\\s*".r
- val symbolRegex3 = "\\s*--.+\\s*".r
- val blankRegex = "\\s*".r
- code.split("\n").foreach {
- case blankRegex() | setRegex() | symbolRegex1() | symbolRegex2() | symbolRegex3() | restartRegex() | kindRegex() =>
- case str => msg ++= str ++ "\n"
- }
- StringUtils.strip(msg.toString())
- }
-
-}
diff --git a/ujes/definedEngines/pipeline/enginemanager/pom.xml b/ujes/definedEngines/pipeline/enginemanager/pom.xml
index 7efdddb..1aea5d5 100644
--- a/ujes/definedEngines/pipeline/enginemanager/pom.xml
+++ b/ujes/definedEngines/pipeline/enginemanager/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala
index cb510de..cce8def 100644
--- a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala
+++ b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala
@@ -18,7 +18,6 @@
import com.webank.wedatasphere.linkis.enginemanager.AbstractEngineCreator
import com.webank.wedatasphere.linkis.enginemanager.process.ProcessEngineBuilder
-import org.springframework.stereotype.Component
/**
* Created by johnnwang on 2018/11/15.
diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala
index dea14b1..8381456 100644
--- a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala
+++ b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.enginemanager.pipeline
-import com.webank.wedatasphere.linkis.common.utils.ClassUtils
import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration
import com.webank.wedatasphere.linkis.enginemanager.pipeline.conf.PipeLineConfiguration
import com.webank.wedatasphere.linkis.enginemanager.process.JavaProcessEngineBuilder
diff --git a/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh b/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh
index 56cab6d..80cc775 100644
--- a/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh
+++ b/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/ujes/definedEngines/pipeline/entrance/pom.xml b/ujes/definedEngines/pipeline/entrance/pom.xml
index ca5f811..cc95e78 100644
--- a/ujes/definedEngines/pipeline/entrance/pom.xml
+++ b/ujes/definedEngines/pipeline/entrance/pom.xml
@@ -21,7 +21,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
</parent>
diff --git a/ujes/definedEngines/python/engine/pom.xml b/ujes/definedEngines/python/engine/pom.xml
index 33b6fba..a088645 100644
--- a/ujes/definedEngines/python/engine/pom.xml
+++ b/ujes/definedEngines/python/engine/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-python-engine</artifactId>
diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala
index 4bb9bce..8768216 100644
--- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala
+++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.engine
-
import com.webank.wedatasphere.linkis.engine.exception.QueryFailedException
import org.json4s.JValue
import org.json4s.JsonAST.JString
diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala
index ed8b179..7712d3e 100644
--- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala
+++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala
@@ -15,6 +15,7 @@
*/
package com.webank.wedatasphere.linkis.engine
+
/**
* Created by allenlliu on 2019/4/8.
*/
diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala
index 9b919b0..4994270 100644
--- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala
+++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala
@@ -16,8 +16,8 @@
package com.webank.wedatasphere.linkis.engine
-import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, PythonCodeParser, SQLCodeParser}
import com.webank.wedatasphere.linkis.engine.execute.hook.{MaxExecuteNumEngineHook, ReleaseEngineHook}
+import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, PythonCodeParser}
import org.slf4j.LoggerFactory
import org.springframework.context.annotation.Bean
import org.springframework.stereotype.Component
diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala
index 86c7763..af707d6 100644
--- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala
+++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala
@@ -23,30 +23,27 @@
import java.io.{File, FileFilter, FileOutputStream, InputStream}
import java.net.ServerSocket
import java.nio.file.Files
-
-import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
-import com.webank.wedatasphere.linkis.engine.exception.{ExecuteException, PythonExecuteError, QueryFailedException}
-import com.webank.wedatasphere.linkis.engine.util.PythonConfiguration._
-import com.webank.wedatasphere.linkis.engine.util._
-import org.apache.commons.exec.CommandLine
-import org.apache.commons.lang.StringUtils
-import py4j.GatewayServer
import java.util.{List => JList}
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.engine.conf.PythonEngineConfiguration
-
-import scala.collection.JavaConverters._
+import com.webank.wedatasphere.linkis.engine.exception.{ExecuteException, PythonExecuteError}
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.rs.RsOutputStream
-import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
+import com.webank.wedatasphere.linkis.engine.util.PythonConfiguration._
+import com.webank.wedatasphere.linkis.engine.util._
import com.webank.wedatasphere.linkis.storage.domain._
-import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter}
import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord}
+import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter}
+import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
+import org.apache.commons.exec.CommandLine
import org.apache.commons.io.IOUtils
+import org.apache.commons.lang.StringUtils
import org.scalactic.Pass
+import py4j.GatewayServer
import scala.collection.JavaConversions._
-import scala.collection.JavaConverters
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, StringBuilder}
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, Future, Promise}
diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala
index ead2014..895d20f 100644
--- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala
+++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala
@@ -16,15 +16,13 @@
package com.webank.wedatasphere.linkis.engine.executors
-import java.util.UUID
-
-import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging}
+import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.PythonSession
import com.webank.wedatasphere.linkis.engine.exception.EngineException
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext}
import com.webank.wedatasphere.linkis.engine.rs.RsOutputStream
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
-import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, LoadResource, Resource}
+import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, Resource}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer._
import org.apache.commons.io.IOUtils
diff --git a/ujes/definedEngines/python/enginemanager/pom.xml b/ujes/definedEngines/python/enginemanager/pom.xml
index 9e623b9..cac5209 100644
--- a/ujes/definedEngines/python/enginemanager/pom.xml
+++ b/ujes/definedEngines/python/enginemanager/pom.xml
@@ -23,17 +23,17 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-python-enginemanager</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
<packaging>jar</packaging>
<!--<version>${bdp.ide.version}</version>-->
<!--<name>linkis-Spark-EngineManager</name>-->
<properties>
- <dwc.python.version>0.9.3</dwc.python.version>
+ <dwc.python.version>0.9.4</dwc.python.version>
</properties>
<dependencies>
diff --git a/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala b/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala
index a8904db..ea57d29 100644
--- a/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala
+++ b/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala
@@ -16,9 +16,7 @@
package com.webank.wedatasphere.linkis.enginemanager.process
-import com.webank.wedatasphere.linkis.enginemanager.EngineHook
import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration
-import com.webank.wedatasphere.linkis.enginemanager.hook.{ConsoleConfigurationEngineHook, JarUdfEngineHook, PyFunctionEngineHook, PyUdfEngineHook}
import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo
import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, ResourceRequestPolicy}
import com.webank.wedatasphere.linkis.rpc.Sender
diff --git a/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh b/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh
index 56cab6d..80cc775 100644
--- a/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh
+++ b/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/ujes/definedEngines/python/entrance/pom.xml b/ujes/definedEngines/python/entrance/pom.xml
index b06dae1..0e1b31a 100644
--- a/ujes/definedEngines/python/entrance/pom.xml
+++ b/ujes/definedEngines/python/entrance/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-python-entracne</artifactId>
diff --git a/ujes/definedEngines/shell/engine/pom.xml b/ujes/definedEngines/shell/engine/pom.xml
index f18edef..f9f2129 100644
--- a/ujes/definedEngines/shell/engine/pom.xml
+++ b/ujes/definedEngines/shell/engine/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala
index 3847a90..322c81b 100644
--- a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala
+++ b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala
@@ -3,15 +3,14 @@
import java.io.{BufferedReader, InputStreamReader}
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext}
+import com.webank.wedatasphere.linkis.engine.shell.conf.ShellEngineConfiguration
+import com.webank.wedatasphere.linkis.engine.shell.exception.ShellCodeErrorException
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, Resource}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer._
-import com.webank.wedatasphere.linkis.engine.shell.conf.ShellEngineConfiguration
-import com.webank.wedatasphere.linkis.engine.shell.exception.ShellCodeErrorException
-import org.apache.commons.lang.StringUtils
-import com.webank.wedatasphere.linkis.engine.execute.EngineExecutor
import org.apache.commons.io.IOUtils
+import org.apache.commons.lang.StringUtils
/**
* created by cooperyang on 2019/5/14
diff --git a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala
index ef92903..81dd094 100644
--- a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala
+++ b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala
@@ -2,8 +2,8 @@
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorFactory}
-import com.webank.wedatasphere.linkis.server.JMap
import com.webank.wedatasphere.linkis.engine.shell.exception.NoCorrectUserException
+import com.webank.wedatasphere.linkis.server.JMap
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component
diff --git a/ujes/definedEngines/shell/enginemanager/pom.xml b/ujes/definedEngines/shell/enginemanager/pom.xml
index 4987448..8e821e2 100644
--- a/ujes/definedEngines/shell/enginemanager/pom.xml
+++ b/ujes/definedEngines/shell/enginemanager/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala b/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala
index 4f95bb7..de397dd 100644
--- a/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala
+++ b/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala
@@ -3,11 +3,10 @@
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.enginemanager.EngineHook
import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration
-import com.webank.wedatasphere.linkis.enginemanager.hook.{ConsoleConfigurationEngineHook, JarLoaderEngineHook}
-import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, ResourceRequestPolicy}
+import com.webank.wedatasphere.linkis.enginemanager.hook.ConsoleConfigurationEngineHook
import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo
+import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, ResourceRequestPolicy}
import com.webank.wedatasphere.linkis.rpc.Sender
-import org.slf4j.{Logger, LoggerFactory}
import org.springframework.context.annotation.{Bean, Configuration}
/**
diff --git a/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh b/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh
index 56cab6d..80cc775 100644
--- a/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh
+++ b/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/ujes/definedEngines/shell/entrance/pom.xml b/ujes/definedEngines/shell/entrance/pom.xml
index 40f9f50..1c6765b 100644
--- a/ujes/definedEngines/shell/entrance/pom.xml
+++ b/ujes/definedEngines/shell/entrance/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/spark/engine/pom.xml b/ujes/definedEngines/spark/engine/pom.xml
index 92fcda8..704c911 100644
--- a/ujes/definedEngines/spark/engine/pom.xml
+++ b/ujes/definedEngines/spark/engine/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-spark-engine</artifactId>
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala
index fc8e763..e015878 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala
@@ -20,14 +20,13 @@
import java.util.concurrent.TimeUnit
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
-import com.webank.wedatasphere.linkis.engine.spark.common.{LineBufferedStream, Starting, State}
+import com.webank.wedatasphere.linkis.engine.spark.common.{LineBufferedStream, Starting, State, _}
import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse}
import org.apache.commons.io.IOUtils
import org.json4s._
-import scala.concurrent.{Await, ExecutionContext, Future}
import scala.concurrent.duration.Duration
-import com.webank.wedatasphere.linkis.engine.spark.common._
+import scala.concurrent.{Await, ExecutionContext, Future}
/**
* Created by allenlliu on 2018/11/19.
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala
index 17bf490..638a89a 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala
@@ -15,24 +15,25 @@
*/
package com.webank.wedatasphere.linkis.engine.Interpreter
-import org.apache.spark.{SparkContext, SparkException}
-import org.json4s.jackson.JsonMethods._
-import org.json4s.jackson.Serialization.write
-import org.json4s.{DefaultFormats, JValue}
-import py4j.GatewayServer
+
import java.io._
import java.nio.file.Files
import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.common.io.FsPath
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
-import com.webank.wedatasphere.linkis.engine.configuration.{SparkConfiguration}
+import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
import com.webank.wedatasphere.linkis.engine.spark.common.LineBufferedStream
import com.webank.wedatasphere.linkis.engine.spark.utils.EngineUtils
import com.webank.wedatasphere.linkis.storage.FSFactory
import org.apache.commons.io.IOUtils
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.{SparkContext, SparkException}
+import org.json4s.jackson.JsonMethods._
+import org.json4s.jackson.Serialization.write
+import org.json4s.{DefaultFormats, JValue}
+import py4j.GatewayServer
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala
index c6d4936..7326e99 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala
@@ -37,4 +37,7 @@
val DOLPHIN_LIMIT_LEN = CommonVars("wds.linkis.dolphin.limit.len",5000)
val MDQ_APPLICATION_NAME = CommonVars("wds.linkis.mdq.application.name", "cloud-datasource")
val SHOW_DF_MAX_RES = CommonVars("wds.linkis.show.df.max.res",Int.MaxValue)
+
+
+ val SPARK_PRE_EXECUTION_HOOKS = CommonVars("wds.linkis.spark.pre.execution.hooks", "com.webank.wedatasphere.linkis.engine.cs.CSSparkPreExecutionHook")
}
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala
index 06786dd..92ed884 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala
@@ -1,37 +1,36 @@
-/*
- * Copyright 2019 WeBank
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.webank.wedatasphere.linkis.engine.configuration
-
-import com.webank.wedatasphere.linkis.engine.condition.EngineHooksCondition
-import com.webank.wedatasphere.linkis.engine.execute.hook._
-import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SparkCombinedCodeParser}
-import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
-import org.springframework.context.annotation.{Bean, Conditional, Configuration}
-
-/**
- * Created by allenlliu on 2018/12/3.
- */
-@Configuration
-class SparkEngineServerSpringConfiguration {
- @Bean(Array("codeParser"))
- def createCodeParser(): CodeParser = new SparkCombinedCodeParser()
-
-
- @Bean(Array("engineHooks"))
- @Conditional(Array(classOf[EngineHooksCondition]))
- def createEngineHooks(): Array[EngineHook] = Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new JarUdfEngineHook, new PyUdfEngineHook, new ScalaUdfEngineHook, new PyFunctionEngineHook, new ScalaFunctionEngineHook)
-}
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.configuration
+
+import com.webank.wedatasphere.linkis.engine.condition.EngineHooksCondition
+import com.webank.wedatasphere.linkis.engine.execute.hook._
+import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SparkCombinedCodeParser}
+import org.springframework.context.annotation.{Bean, Conditional, Configuration}
+
+/**
+ * Created by allenlliu on 2018/12/3.
+ */
+@Configuration
+class SparkEngineServerSpringConfiguration {
+ @Bean(Array("codeParser"))
+ def createCodeParser(): CodeParser = new SparkCombinedCodeParser()
+
+
+ @Bean(Array("engineHooks"))
+ @Conditional(Array(classOf[EngineHooksCondition]))
+ def createEngineHooks(): Array[EngineHook] = Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new JarUdfEngineHook, new PyUdfEngineHook, new ScalaUdfEngineHook, new PyFunctionEngineHook, new ScalaFunctionEngineHook)
+}
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkHelper.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkHelper.scala
new file mode 100644
index 0000000..287e1c9
--- /dev/null
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkHelper.scala
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.cs
+
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
+import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import org.apache.spark.SparkContext
+
+/**
+ * @author peacewong
+ * @date 2020/4/19 16:39
+ */
+object CSSparkHelper {
+
+ def setContextIDInfoToSparkConf(engineExecutorContext: EngineExecutorContext, sparkContext: SparkContext): Unit = {
+ val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties)
+ val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties)
+ sparkContext.setLocalProperty(CSCommonUtils.CONTEXT_ID_STR, contextIDValueStr)
+ sparkContext.setLocalProperty(CSCommonUtils.NODE_NAME_STR, nodeNameStr)
+ }
+
+}
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkPreExecutionHook.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkPreExecutionHook.scala
new file mode 100644
index 0000000..b575843
--- /dev/null
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkPreExecutionHook.scala
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.cs
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils
+import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import com.webank.wedatasphere.linkis.engine.extension.SparkPreExecutionHook
+import javax.annotation.PostConstruct
+import org.springframework.stereotype.Component
+
+/**
+ * @author peacewong
+ * @date 2020/3/6 0:06
+ */
+@Component
+class CSSparkPreExecutionHook extends SparkPreExecutionHook with Logging{
+
+ @PostConstruct
+ def init(): Unit ={
+ SparkPreExecutionHook.register(this)
+ }
+
+ private val csTableParser = new CSTableParser
+
+ override def hookName: String = "CSSparkPreExecutionHook"
+
+ override def callPreExecutionHook(engineExecutorContext: EngineExecutorContext, code: String): String = {
+
+ var parsedCode = code
+ val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties)
+ val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties)
+ info(s"Start to call CSSparkPreExecutionHook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr")
+ parsedCode = try {
+ csTableParser.parse(engineExecutorContext, parsedCode, contextIDValueStr, nodeNameStr)
+ } catch {
+ case t: Throwable =>
+ info("Failed to parser cs table", t)
+ parsedCode
+ }
+ info(s"Finished to call CSSparkPreExecutionHook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr")
+ parsedCode
+ }
+}
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableParser.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableParser.scala
new file mode 100644
index 0000000..00580ef
--- /dev/null
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableParser.scala
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.cs
+
+import java.util.regex.Pattern
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.client.service.CSTableService
+import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
+import com.webank.wedatasphere.linkis.engine.exception.ExecuteError
+import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import org.apache.commons.lang.StringUtils
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.execution.datasources.csv.DolphinToSpark
+
+import scala.collection.mutable.ArrayBuffer
+
+/**
+ * @author peacewong
+ * @date 2020/3/6 9:57
+ */
+class CSTableParser extends Logging {
+
+ private val pb: Pattern = Pattern.compile(CSCommonUtils.CS_TMP_TABLE_PREFIX + "[^\\s\";'()]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE)
+
+ private val DB = "default."
+
+
+ private def getCSTempTable(code: String): Array[String] = {
+ val bmlResourceNames = new ArrayBuffer[String]()
+ val mb = pb.matcher(code)
+ while (mb.find) bmlResourceNames.append(mb.group.trim)
+ bmlResourceNames.toArray
+ }
+
+ /**
+ * 1. code parse cs_tamp
+ * 2. getCSTable
+ * 3. registerTable：暂时用dopphin，后续修改为拼接sql语句
+ *
+ * @param engineExecutorContext
+ * @param code
+ * @param contextIDValueStr
+ * @param nodeNameStr
+ * @return
+ */
+ def parse(engineExecutorContext: EngineExecutorContext, code: String, contextIDValueStr: String, nodeNameStr: String): String = {
+ val csTempTables = getCSTempTable(code)
+ val parsedTables = new ArrayBuffer[String]()
+ csTempTables.foreach{ csTempTable =>
+ val table = getCSTable(csTempTable, contextIDValueStr, nodeNameStr)
+ if (null == table){
+ throw new ExecuteError(40007,s"The csTable that name is $csTempTable not found in cs")
+ }
+ registerTempTable(table)
+ parsedTables.append(csTempTable)
+ }
+ StringUtils.replaceEach(code, csTempTables,parsedTables.toArray)
+ }
+
+ /**
+ * TODO peaceWong From cs to get csTable
+ * Exact Match
+ * @param csTempTable
+ * @return
+ */
+ def getCSTable(csTempTable:String, contextIDValueStr: String, nodeNameStr: String):CSTable = {
+ CSTableService.getInstance().getUpstreamSuitableTable(contextIDValueStr, nodeNameStr, csTempTable)
+ }
+
+ def registerTempTable(csTable: CSTable):Unit = {
+ val spark = SparkSession.builder().enableHiveSupport().getOrCreate()
+ info(s"Start to create tempView to sparkSession viewName(${csTable.getName}) location(${csTable.getLocation})")
+ DolphinToSpark.createTempView(spark, csTable.getName, csTable.getLocation, true)
+ info(s"Finished to create tempView to sparkSession viewName(${csTable.getName}) location(${csTable.getLocation})")
+ }
+}
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala
index 37c69d0..60c4f6f 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala
@@ -15,6 +15,7 @@
*/
package com.webank.wedatasphere.linkis.engine.execute
+
import com.webank.wedatasphere.linkis.protocol.engine.RequestTask
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, JobExecuteRequest, RunTypeExecuteRequest}
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala
index afe3dfd..a1c11cc 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala
@@ -22,8 +22,9 @@
import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging, Utils}
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
+import com.webank.wedatasphere.linkis.engine.cs.CSSparkHelper
import com.webank.wedatasphere.linkis.engine.exception.{NoSupportEngineException, SparkEngineException}
-import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext}
+import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext, SparkCombinedCodeParser}
import com.webank.wedatasphere.linkis.engine.extension.{SparkPostExecutionHook, SparkPreExecutionHook}
import com.webank.wedatasphere.linkis.engine.spark.common._
import com.webank.wedatasphere.linkis.engine.spark.utils.{EngineUtils, JobProgressUtil}
@@ -64,6 +65,7 @@
}
}
}, 1000, 3000, TimeUnit.MILLISECONDS)
+ setCodeParser(new SparkCombinedCodeParser)
super.init()
}
@@ -225,15 +227,26 @@
// get field names
//logger.info("SCHEMA BEGIN")
import java.util
+
+ import scala.collection.JavaConversions._
val colSet = new util.HashSet[String]()
val schema = dataFrame.schema
var columnsSet:StructType = null
schema foreach (s => colSet.add(s.name))
if (colSet.size() < schema.size){
val arr:ArrayBuffer[StructField] = new ArrayBuffer[StructField]()
+ val tmpSet = new util.HashSet[StructField]()
dataFrame.queryExecution.analyzed.output foreach {
attri => val tempAttri = StructField(attri.qualifiedName, attri.dataType, attri.nullable, attri.metadata)
- arr += tempAttri
+ tmpSet += tempAttri
+ }
+ if (tmpSet.size() < schema.size){
+ dataFrame.queryExecution.analyzed.output foreach {
+ attri => val tempAttri = StructField(attri.toString(), attri.dataType, attri.nullable, attri.metadata)
+ arr += tempAttri
+ }
+ }else{
+ tmpSet.foreach(arr += _)
}
columnsSet = StructType(arr.toArray)
}else{
@@ -242,6 +255,7 @@
//val columnsSet = dataFrame.schema
val columns = columnsSet.map(c =>
Column(c.name, DataType.toDataType(c.dataType.typeName.toLowerCase), c.getComment().orNull)).toArray[Column]
+ columns.foreach(c => info(s"c is ${c.columnName}, comment is ${c.comment}"))
if (columns == null || columns.isEmpty) return
val metaData = new TableMetaData(columns)
val writer = if (StringUtils.isNotBlank(alias))
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala
index 1a9c565..97832b6 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala
@@ -19,12 +19,9 @@
import java.io.IOException
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext}
+import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.spark.common.Kind
-import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.SQLContext
/**
* Created by allenlliu on 2019/4/8.
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala
index 65c5b43..66fd929 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala
@@ -23,29 +23,28 @@
import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.engine.Interpreter.PythonInterpreter._
-import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
+import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._
import com.webank.wedatasphere.linkis.engine.exception.ExecuteError
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
import com.webank.wedatasphere.linkis.engine.imexport.CsvRelation
import com.webank.wedatasphere.linkis.engine.rs.RsOutputStream
import com.webank.wedatasphere.linkis.engine.spark.common.PySpark
import com.webank.wedatasphere.linkis.engine.spark.utils.EngineUtils
-import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse}
+import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter}
-import org.apache.commons.io.IOUtils
import org.apache.commons.exec.CommandLine
+import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils
+import org.apache.spark.SparkContext
import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession}
-import org.apache.spark.{SparkContext, SparkException}
import py4j.GatewayServer
import scala.collection.JavaConversions._
import scala.collection.mutable.StringBuilder
-import scala.concurrent.duration.Duration
import scala.concurrent._
+import scala.concurrent.duration.Duration
/**
* Created by allenlliu on 2018/11/19.
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala
index f744236..02a89bd 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala
@@ -19,24 +19,19 @@
import java.io.File
import java.lang.reflect.Constructor
+import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._
import com.webank.wedatasphere.linkis.engine.exception.SparkSessionNullException
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorFactory
import com.webank.wedatasphere.linkis.engine.executors.{SparkEngineExecutor, SparkPythonExecutor, SparkScalaExecutor, SparkSqlExecutor}
-import com.webank.wedatasphere.linkis.engine.spark.utils.EngineUtils
import com.webank.wedatasphere.linkis.server.JMap
import org.apache.commons.lang.StringUtils
import org.apache.spark.sql.{SQLContext, SparkSession}
+import org.apache.spark.util.SparkUtils
import org.apache.spark.{SparkConf, SparkContext}
import org.springframework.stereotype.Component
-import com.google.common.base.Joiner
-import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType}
-import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
-import org.apache.spark.util.SparkUtils
-
-import scala.collection.mutable.ArrayBuffer
-import scala.collection.JavaConversions._
/**
* Created by allenlliu on 2019/4/8.
*/
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala
index 31046e8..cb9e4ea 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala
@@ -22,7 +22,7 @@
import java.text.SimpleDateFormat
import java.util.Locale
-import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.imexport.util.ImExportUtils
import org.apache.hadoop.fs.Path
import org.apache.hadoop.io.{IOUtils, LongWritable, Text}
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala
index bace449..d04574f 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala
@@ -15,8 +15,6 @@
*/
package com.webank.wedatasphere.linkis.engine.mdq
-import javax.annotation.PostConstruct
-
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
@@ -26,6 +24,7 @@
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse}
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import javax.annotation.PostConstruct
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala
index e5504fc..de552bc 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.engine.mdq
import java.util
-import javax.annotation.PostConstruct
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
@@ -27,6 +26,7 @@
import com.webank.wedatasphere.linkis.protocol.mdq.{DDLRequest, DDLResponse}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import javax.annotation.PostConstruct
import org.apache.commons.lang.StringUtils
import org.springframework.stereotype.Component
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala
index f2b6a85..00f8781 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala
@@ -21,7 +21,7 @@
import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
-import com.webank.wedatasphere.linkis.engine.configuration.{SparkConfiguration}
+import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
import org.apache.commons.io.IOUtils
import scala.concurrent.duration.Duration
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala
index 4e1d8ec..a4ab282 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala
@@ -17,8 +17,8 @@
package com.webank.wedatasphere.linkis.engine.spark.common
import scala.collection.Iterable
-import scala.collection.mutable.ArrayBuffer
import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
/**
* Created by allenlliu on 2018/11/19.
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala
index 4c4e333..758473f 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala
@@ -27,9 +27,9 @@
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._
import com.webank.wedatasphere.linkis.engine.spark.common.LineBufferedProcess
import com.webank.wedatasphere.linkis.rpc.Sender
-import com.webank.wedatasphere.linkis.storage.{FSFactory, LineMetaData}
import com.webank.wedatasphere.linkis.storage.resultset.ResultSetReader
import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import com.webank.wedatasphere.linkis.storage.{FSFactory, LineMetaData}
import scala.util.Random
diff --git a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala
index 7db344a..3393bc9 100644
--- a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala
+++ b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala
@@ -16,7 +16,6 @@
package org.apache.spark.sql.execution.datasources.csv
-
import java.util
import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration
@@ -60,7 +59,7 @@
def toSparkType(dataType:wds.DataType):DataType = dataType match {
case wds.NullType => NullType
- //case dwc.StringType | dwc.CharType | dwc.VarcharType | dwc.StructType | dwc.ListType | dwc.ArrayType | dwc.MapType => StringType
+ //case wds.StringType | wds.CharType | wds.VarcharType | wds.StructType | wds.ListType | wds.ArrayType | wds.MapType => StringType
case wds.BooleanType => BooleanType
case wds.ShortIntType => ShortType
case wds.IntType => IntegerType
@@ -69,7 +68,7 @@
case wds.DoubleType => DoubleType
case wds.DecimalType => DecimalType(bigDecimalPrecision,bigDecimalScale)
case wds.DateType => DateType
- case wds.TimestampType => TimestampType
+ //case wds.TimestampType => TimestampType
case wds.BinaryType => BinaryType
case _ => StringType
}
diff --git a/ujes/definedEngines/spark/enginemanager/pom.xml b/ujes/definedEngines/spark/enginemanager/pom.xml
index d10aec3..ae8a3b4 100644
--- a/ujes/definedEngines/spark/enginemanager/pom.xml
+++ b/ujes/definedEngines/spark/enginemanager/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-spark-enginemanager</artifactId>
diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala
index 320cec6..1f6a289 100644
--- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala
+++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala
@@ -16,12 +16,10 @@
package com.webank.wedatasphere.linkis.enginemanager.configuration
-
import com.webank.wedatasphere.linkis.common.conf.{CommonVars, Configuration}
import com.webank.wedatasphere.linkis.common.utils.{ClassUtils, Logging}
import com.webank.wedatasphere.linkis.engine.factory.SparkEngineExecutorFactory
import com.webank.wedatasphere.linkis.enginemanager.AbstractEngineCreator
-import com.webank.wedatasphere.linkis.enginemanager.process.{JavaProcessEngineBuilder, SparkSubmitProcessBuilder}
import scala.collection.mutable.ArrayBuffer
/**
diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala
index af08513..cb2bb5a 100644
--- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala
+++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala
@@ -22,7 +22,6 @@
import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo
import com.webank.wedatasphere.linkis.resourcemanager.{DriverAndYarnResource, LoadInstanceResource, ResourceRequestPolicy}
import com.webank.wedatasphere.linkis.rpc.Sender
-import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.context.annotation.{Bean, Configuration}
/**
diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala
index 1f338e8..5aaef2f 100644
--- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala
+++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala
@@ -18,11 +18,8 @@
import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser
import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration
+import com.webank.wedatasphere.linkis.enginemanager.AbstractEngineCreator
import com.webank.wedatasphere.linkis.enginemanager.impl.UserTimeoutEngineResource
-import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineCreator, Engine, EngineManagerReceiver, EngineResource}
-import com.webank.wedatasphere.linkis.protocol.engine.{EngineCallback, RequestEngine}
-import com.webank.wedatasphere.linkis.rpc.Sender
import org.springframework.stereotype.Component
/**
diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala
index 99bac97..ae678d1 100644
--- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala
+++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala
@@ -19,7 +19,7 @@
import java.lang.ProcessBuilder.Redirect
-import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging}
+import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.enginemanager.EngineResource
import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration._
import com.webank.wedatasphere.linkis.enginemanager.configuration.SparkConfiguration
@@ -29,7 +29,6 @@
import com.webank.wedatasphere.linkis.enginemanager.process.SparkSubmitProcessBuilder.{AbsolutePath, Path, RelativePath}
import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine
import com.webank.wedatasphere.linkis.resourcemanager.DriverAndYarnResource
-import org.apache.commons.lang.StringUtils
import scala.collection.mutable.ArrayBuffer
@@ -135,7 +134,7 @@
this.deployMode("client")
val driverJavaSet = "\"-Dwds.linkis.configuration=linkis-engine.properties " + SparkConfiguration.getJavaRemotePort + "\""
this.conf(SPARK_DRIVER_EXTRA_JAVA_OPTIONS.key, driverJavaSet)
- this.name(properties.getOrDefault("appName", "sparksqltest"))
+ this.name(properties.getOrDefault("appName", "linkis"))
this.className(properties.getOrDefault("className", "com.webank.wedatasphere.linkis.engine.DataWorkCloudEngineApplication"))
properties.getOrDefault("archives", "").toString.split(",").map(RelativePath).foreach(this.archive)
this.driverCores(DWC_SPARK_DRIVER_CORES)
@@ -153,7 +152,7 @@
this.driverClassPath(SPARK_DRIVER_CLASSPATH.getValue)
this.redirectOutput(Redirect.PIPE)
this.redirectErrorStream(true)
- this.env("spark.app.name", properties.getOrDefault("appName", "dwc" + request.creator))
+ this.env("spark.app.name", properties.getOrDefault("appName", "linkis" + request.creator))
}
diff --git a/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh b/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh
index 56cab6d..80cc775 100644
--- a/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh
+++ b/ujes/definedEngines/spark/entrance/bin/start-spark-entrance.sh
@@ -26,7 +26,7 @@
fi
fi
-nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
+nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out &
pid=$!
if [[ -z "${pid}" ]]; then
echo "server $SERVER_NAME start failed!"
diff --git a/ujes/definedEngines/spark/entrance/pom.xml b/ujes/definedEngines/spark/entrance/pom.xml
index 7641bd5..070592a 100644
--- a/ujes/definedEngines/spark/entrance/pom.xml
+++ b/ujes/definedEngines/spark/entrance/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-spark-entracne</artifactId>
diff --git a/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala b/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala
index 500cb40..c522d64 100644
--- a/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala
+++ b/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.entrance.job
-
/**
* Created by allenlliu on 2018/12/5.
*/
diff --git a/ujes/definedEngines/tispark/engine/pom.xml b/ujes/definedEngines/tispark/engine/pom.xml
index 0972352..0e334bb 100644
--- a/ujes/definedEngines/tispark/engine/pom.xml
+++ b/ujes/definedEngines/tispark/engine/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala
index 75ab540..664308d 100644
--- a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala
+++ b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala
@@ -16,11 +16,6 @@
package com.webank.wedatasphere.linkis.tispark.engine.configuration
-import com.webank.wedatasphere.linkis.engine.execute.EngineHook
-import com.webank.wedatasphere.linkis.engine.execute.hook._
-import com.webank.wedatasphere.linkis.tispark.engine.hook.{TiSparkHook, UserDataBaseHook}
-import org.springframework.context.annotation.{Bean, Configuration}
-
/**
* Created by johnnwang on 2019/6/26.
*/
diff --git a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala
index d1eafba..7c8177b 100644
--- a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala
+++ b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala
@@ -16,13 +16,6 @@
package com.webank.wedatasphere.linkis.tispark.engine.hook
-import com.webank.wedatasphere.linkis.common.utils.Logging
-import com.webank.wedatasphere.linkis.engine.exception.EngineErrorException
-import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook}
-import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest}
-import com.webank.wedatasphere.linkis.server.JMap
-import org.apache.commons.lang.StringUtils
-
/**
* Created by johnnwang on 2019/6/26.
*/
diff --git a/ujes/engine/pom.xml b/ujes/engine/pom.xml
index 387ce2d..3b52059 100644
--- a/ujes/engine/pom.xml
+++ b/ujes/engine/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-engine</artifactId>
@@ -80,6 +80,11 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-ujes-client</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
</dependencies>
diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java
index adcc6cd..63b67fb 100644
--- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java
+++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java
@@ -17,8 +17,6 @@
package com.webank.wedatasphere.linkis.engine.log;
-import org.apache.logging.log4j.core.LogEvent;
-
import java.util.List;
/**
diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java
index fb89702..5512ce5 100644
--- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java
+++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java
@@ -18,8 +18,6 @@
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration$;
import org.apache.commons.lang.StringUtils;
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.core.LogEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,7 +119,7 @@
}
@Override
- public List<String> getRemain() {
+ public synchronized List<String> getRemain() {
return logs.getRemain();
}
diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java
index d418ba4..e7482f6 100644
--- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java
+++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java
@@ -22,18 +22,10 @@
*/
package com.webank.wedatasphere.linkis.engine.log;
-import com.webank.wedatasphere.linkis.engine.EngineReceiver;
import com.webank.wedatasphere.linkis.rpc.Sender;
import com.webank.wedatasphere.linkis.scheduler.listener.LogListener;
-import com.webank.wedatasphere.linkis.scheduler.queue.Job;
-import com.webank.wedatasphere.linkis.server.Message;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.spi.LoggingEvent;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import java.util.HashMap;
-import java.util.Map;
public class RPCAppender extends AppenderSkeleton {
diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java
index 3b47153..2fe1c88 100644
--- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java
+++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java
@@ -20,6 +20,7 @@
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration;
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration$;
import com.webank.wedatasphere.linkis.scheduler.listener.LogListener;
+import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.Filter;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.LogEvent;
@@ -51,25 +52,33 @@
private LogCache logCache;
private static final Logger logger = LoggerFactory.getLogger(SendAppender.class);
+ private static final String IGNORE_WORDS = EngineConfiguration.ENGINE_IGNORE_WORDS().getValue();
+
+ private static final String[] IGNORE_WORD_ARR = IGNORE_WORDS.split(",");
+
+ private static final String PASS_WORDS = EngineConfiguration.ENGINE_PASS_WORDS().getValue();
+
+ private static final String[] PASS_WORDS_ARR = PASS_WORDS.split(",");
+
class SendThread implements Runnable{
@Override
public void run() {
- if (logListener == null){
+ if (logListener == null){
//ignore
- }else{
- if (logCache == null){
- logger.warn("logCache is null");
- return;
- }
- List<String> logs = logCache.getRemain();
- if (logs.size() > 0){
- StringBuilder sb = new StringBuilder();
- for(String log : logs){
- sb.append(log);
- }
- logListener.onLogUpdate(null, sb.toString());
- }
- }
+ }else{
+ if (logCache == null){
+ logger.warn("logCache is null");
+ return;
+ }
+ List<String> logs = logCache.getRemain();
+ if (logs.size() > 0){
+ StringBuilder sb = new StringBuilder();
+ for(String log : logs){
+ sb.append(log).append("\n");
+ }
+ logListener.onLogUpdate(null, sb.toString());
+ }
+ }
}
}
@@ -77,8 +86,7 @@
public SendAppender(final String name, final Filter filter, final Layout<? extends Serializable> layout,
final boolean ignoreExceptions) {
super(name, filter, layout, ignoreExceptions);
- //todo enjoyyin 500 to be made configurable ide number（500要做成可配置ide数字）
- //this.logCache = new MountLogCache((Integer) EngineConfiguration.ENGINE_LOG_CACHE_NUM().getValue());
+ //todo cooperyang 500要做成可配置ide数字
this.logCache = LogHelper.logCache();
SendThread thread = new SendThread();
Utils.defaultScheduler().scheduleAtFixedRate(thread, 10, (Integer)EngineConfiguration$.MODULE$.ENGINE_LOG_SEND_TIME_INTERVAL().getValue(), TimeUnit.MILLISECONDS);
@@ -88,16 +96,34 @@
logListener = ll;
}
-// public static void setLogCache(LogCache lc){
-// logCache = lc;
-// }
+
@Override
public void append(LogEvent event) {
if (logListener == null) {
return;
}
- logCache.cacheLog(new String(getLayout().toByteArray(event)));
+ String logStr = new String(getLayout().toByteArray(event));
+ if (event.getLevel().intLevel() == Level.INFO.intLevel()){
+ boolean flag = false;
+ for(String ignoreLog : IGNORE_WORD_ARR){
+ if (logStr.contains(ignoreLog)){
+ flag = true;
+ break;
+ }
+ }
+ for(String word : PASS_WORDS_ARR){
+ if(logStr.contains(word)){
+ flag = false;
+ break;
+ }
+ }
+ if (!flag) {
+ logCache.cacheLog(logStr);
+ }
+ }else{
+ logCache.cacheLog(logStr);
+ }
}
@PluginFactory
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala
index 255ac59..006bad7 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala
@@ -17,7 +17,7 @@
package com.webank.wedatasphere.linkis.engine
import java.lang.management.ManagementFactory
-import java.util.concurrent.{Future, ScheduledFuture, TimeUnit}
+import java.util.concurrent.{Future, TimeUnit}
import com.webank.wedatasphere.linkis.common.ServiceInstance
import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser
@@ -28,7 +28,7 @@
import com.webank.wedatasphere.linkis.engine.exception.{EngineErrorException, JobNotExistsException}
import com.webank.wedatasphere.linkis.engine.execute.scheduler.EngineGroupFactory
import com.webank.wedatasphere.linkis.engine.execute.{CommonEngineJob, _}
-import com.webank.wedatasphere.linkis.engine.log.{LogHelper, MountLogCache, SendAppender}
+import com.webank.wedatasphere.linkis.engine.log.{LogHelper, SendAppender}
import com.webank.wedatasphere.linkis.protocol.UserWithCreator
import com.webank.wedatasphere.linkis.protocol.engine._
import com.webank.wedatasphere.linkis.resourcemanager.UserResultResource
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala
index 0ab3454..d5ffb82 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala
@@ -19,7 +19,6 @@
import java.io.Closeable
import com.webank.wedatasphere.linkis.common.utils.Logging
-import javax.annotation.PreDestroy
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala
index bec668c..f873e83 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala
@@ -26,7 +26,6 @@
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.SchedulerContext
import com.webank.wedatasphere.linkis.scheduler.queue.Job
-import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
import org.springframework.context.annotation.{Bean, Configuration}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala
index 42b2c24..69e59b7 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.engine
import com.webank.wedatasphere.linkis.scheduler.SchedulerContext
-import com.webank.wedatasphere.linkis.scheduler.executer.Executor
/**
* Created by enjoyyin on 2018/9/3.
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PropertiesExecuteRequest.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PropertiesExecuteRequest.scala
new file mode 100644
index 0000000..790e037
--- /dev/null
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PropertiesExecuteRequest.scala
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine
+
+/**
+ * @author peacewong
+ * @date 2020/3/5 17:28
+ */
+trait PropertiesExecuteRequest {
+ val properties: java.util.Map[String, Object]
+}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala
index 52d2998..9c1dfd6 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala
@@ -5,5 +5,5 @@
* Description:
*/
trait ResourceExecuteRequest {
- val resources:java.util.List[Object]
+ def resources:java.util.List[Object]
}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala
index 3de772f..25e5e7f 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala
@@ -60,4 +60,10 @@
val ENGINE_PUSH_PROGRESS_TO_ENTRANCE = CommonVars("wds.linkis.engine.push.progress.enable", true)
val ENGINE_PRE_EXECUTE_HOOK_CLASSES = CommonVars("wds.linkis.engine.pre.hook.class", "com.webank.wedatasphere.linkis.bml.hook.BmlEnginePreExecuteHook")
+
+ val ENGINE_IGNORE_WORDS = CommonVars("wds.linkis.engine.ignore.words", "org.apache.spark.deploy.yarn.Client")
+
+ val ENGINE_PASS_WORDS = CommonVars("wds.linkis.engine.pass.words", "org.apache.hadoop.hive.ql.exec.Task")
+
+ val ENGINE_TUNING_DX_PERIOD = CommonVars("wds.linkis.engine.tuning.dx.period", 1000*60*5)
}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSEnginePreExecuteHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSEnginePreExecuteHook.scala
new file mode 100644
index 0000000..ac08532
--- /dev/null
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSEnginePreExecuteHook.scala
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.cs
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
+import com.webank.wedatasphere.linkis.engine.PropertiesExecuteRequest
+import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import com.webank.wedatasphere.linkis.engine.extension.EnginePreExecuteHook
+import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest
+
+/**
+ * @author peacewong
+ * @date 2020/3/5 17:13
+ */
+class CSEnginePreExecuteHook extends EnginePreExecuteHook with Logging {
+
+ private val csResourceParser: CSResourceParser = new CSResourceParser
+
+ override val hookName: String = "ContextServicePreHook"
+
+
+ override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String = executeRequest match {
+ case propertiesExecuteRequest: PropertiesExecuteRequest =>
+ var parsedCode = code
+ val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(propertiesExecuteRequest.properties)
+ val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(propertiesExecuteRequest.properties)
+ engineExecutorContext.addProperty(CSCommonUtils.CONTEXT_ID_STR, contextIDValueStr)
+ engineExecutorContext.addProperty(CSCommonUtils.NODE_NAME_STR, nodeNameStr)
+ info(s"Start to call cs engine pre hook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr")
+ parsedCode = csResourceParser.parse(propertiesExecuteRequest, parsedCode, contextIDValueStr, nodeNameStr)
+
+ info(s"Finished to call cs engine pre hook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr")
+ parsedCode
+ case _ => code
+ }
+}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala
new file mode 100644
index 0000000..df09dfc
--- /dev/null
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.cs
+
+
+import java.util
+import java.util.regex.Pattern
+
+import com.webank.wedatasphere.linkis.cs.client.service.CSResourceService
+import com.webank.wedatasphere.linkis.engine.PropertiesExecuteRequest
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
+
+/**
+ * @author peacewong
+ * @date 2020/3/5 19:42
+ */
+class CSResourceParser {
+
+ private val pb = Pattern.compile("cs://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE)
+
+ private val PREFIX = "cs://"
+
+ private def getPreFixResourceNames(code: String): Array[String] = {
+ val bmlResourceNames = new ArrayBuffer[String]()
+ val mb = pb.matcher(code)
+ while (mb.find) bmlResourceNames.append(mb.group.trim)
+ bmlResourceNames.toArray
+ }
+
+ def parse(executeRequest: PropertiesExecuteRequest, code: String, contextIDValueStr: String, nodeNameStr: String): String = {
+
+ //TODO getBMLResource peaceWong
+ val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr)
+
+ val parsedResources = new util.ArrayList[util.Map[String, Object]]()
+ val preFixResourceNames = getPreFixResourceNames(code)
+
+ val preFixNames = new ArrayBuffer[String]()
+ val parsedNames = new ArrayBuffer[String]()
+ preFixResourceNames.foreach { preFixResourceName =>
+ val resourceName = preFixResourceName.replace(PREFIX, "").trim
+ val bmlResourceOption = bmlResourceList.find(_.getDownloadedFileName.equals(resourceName))
+ if (bmlResourceOption.isDefined) {
+ val bmlResource = bmlResourceOption.get
+ val map = new util.HashMap[String, Object]()
+ map.put("resourceId", bmlResource.getResourceId)
+ map.put("version", bmlResource.getVersion)
+ map.put("fileName", resourceName)
+ parsedResources.add(map)
+ preFixNames.append(preFixResourceName)
+ parsedNames.append(resourceName)
+ }
+
+ }
+ executeRequest.properties.put("resources", parsedResources)
+ StringUtils.replaceEach(code, preFixNames.toArray, parsedNames.toArray)
+ }
+
+}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala
new file mode 100644
index 0000000..e6c4dad
--- /dev/null
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.engine.cs
+
+import java.util.Date
+
+import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetWriter
+import com.webank.wedatasphere.linkis.common.io.{MetaData, Record}
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.client.service.CSTableService
+import com.webank.wedatasphere.linkis.cs.client.utils.{ContextServiceUtils, SerializeHelper}
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.{ContextScope, ContextType}
+import com.webank.wedatasphere.linkis.cs.common.entity.metadata.{CSColumn, CSTable}
+import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
+import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext
+import com.webank.wedatasphere.linkis.storage.domain.Column
+import com.webank.wedatasphere.linkis.storage.utils.StorageUtils
+import org.apache.commons.lang.StringUtils
+
+/**
+ * @author peacewong
+ * @date 2020/3/13 16:29
+ */
+object CSTableRegister extends Logging{
+
+ def registerTempTable(engineExecutorContext: EngineExecutorContext,
+ writer: ResultSetWriter[_ <: MetaData, _ <: Record], alias: String, columns: Array[Column]): Unit = {
+
+ val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties)
+ val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties)
+
+ if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) {
+ info(s"Start to register TempTable nodeName:$nodeNameStr")
+ writer.flush()
+ val tableName = if (StringUtils.isNotBlank(alias)) s"${CSCommonUtils.CS_TMP_TABLE_PREFIX}${nodeNameStr}_${alias}" else {
+ var i = 1;
+ var rsName: String = null;
+ while (StringUtils.isEmpty(rsName)) {
+ val tmpTable = s"${CSCommonUtils.CS_TMP_TABLE_PREFIX}${nodeNameStr}_rs${i}"
+ i = i + 1
+ val contextKey = new CommonContextKey
+ contextKey.setContextScope(ContextScope.FRIENDLY)
+ contextKey.setContextType(ContextType.METADATA)
+ contextKey.setKey(CSCommonUtils.getTableKey(nodeNameStr, tmpTable))
+ val table = CSTableService.getInstance().getCSTable(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey))
+ if (null == table) {
+ rsName = tmpTable
+ }
+ }
+ rsName
+ }
+ val csTable = new CSTable
+ csTable.setName(tableName)
+ csTable.setAlias(alias)
+ csTable.setAvailable(true)
+ csTable.setComment("cs temp table")
+ csTable.setCreateTime(new Date())
+ csTable.setCreator(StorageUtils.getJvmUser)
+ csTable.setExternalUse(true)
+ csTable.setImport(false)
+ csTable.setLocation(writer.toString)
+ csTable.setPartitionTable(false)
+ csTable.setView(true)
+ val csColumns = columns.map { column =>
+ val csColumn = new CSColumn
+ csColumn.setName(column.columnName)
+ csColumn.setType(column.dataType.typeName)
+ csColumn.setComment(column.comment)
+ csColumn
+ }
+ csTable.setColumns(csColumns)
+ val contextKey = new CommonContextKey
+ contextKey.setContextScope(ContextScope.PUBLIC)
+ contextKey.setContextType(ContextType.METADATA)
+ contextKey.setKey(CSCommonUtils.getTableKey(nodeNameStr, tableName))
+ CSTableService.getInstance().putCSTable(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey), csTable)
+ info(s"Finished to register TempTable nodeName:$nodeNameStr")
+ }
+ }
+}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala
index 3652e31..e60b6e2 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala
@@ -23,7 +23,6 @@
import org.apache.commons.lang.StringUtils
import org.slf4j.{Logger, LoggerFactory}
-import scala.collection.immutable.HashSet
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
@@ -101,8 +100,11 @@
statementBuffer.append(l)
recordBrackets(bracketStack, l)
case l if quotationMarks => statementBuffer.append(l)
- recordBrackets(bracketStack, l)
+ //shanhuang 用于修复python的引号问题
+ //recordBrackets(bracketStack, l)
case l if notDoc && l.startsWith("#") =>
+ case l if StringUtils.isNotBlank(statementBuffer.last) && statementBuffer.last.endsWith("""\""") =>
+ statementBuffer.append(l)
case l if notDoc && l.startsWith(" ") =>
statementBuffer.append(l)
recordBrackets(bracketStack, l.trim)
@@ -180,13 +182,13 @@
if (StringUtils.contains(code, separator)) {
StringUtils.split(code, ";").foreach{
case s if StringUtils.isBlank(s) =>
- case s if isSelectCmdNoLimit(s) => appendStatement(s + " limit " + defaultLimit);
+ case s if isSelectCmdNoLimit(s) => appendStatement(s);
case s => appendStatement(s);
}
} else {
code match {
case s if StringUtils.isBlank(s) =>
- case s if isSelectCmdNoLimit(s) => appendStatement(s + " limit " + defaultLimit);
+ case s if isSelectCmdNoLimit(s) => appendStatement(s);
case s => appendStatement(s);
}
}
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala
index 95c4107..83b7f18 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala
@@ -16,10 +16,8 @@
package com.webank.wedatasphere.linkis.engine.execute
-import com.webank.wedatasphere.linkis.common.log.LogUtils
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration
-import com.webank.wedatasphere.linkis.engine.exception.EngineErrorException
import com.webank.wedatasphere.linkis.engine.extension.EnginePreExecuteHook
import com.webank.wedatasphere.linkis.resourcemanager.Resource
import com.webank.wedatasphere.linkis.scheduler.executer._
@@ -49,14 +47,14 @@
val hooks = new ArrayBuffer[EnginePreExecuteHook]()
EngineConfiguration.ENGINE_PRE_EXECUTE_HOOK_CLASSES.getValue.split(",") foreach {
hookStr => Utils.tryCatch{
- val clazz = Class.forName(hookStr)
+ val clazz = Class.forName(hookStr.trim)
val obj = clazz.newInstance()
obj match {
case hook:EnginePreExecuteHook => hooks += hook
case _ => logger.warn(s"obj is not a engineHook obj is ${obj.getClass}")
}
}{
- case e:Exception => logger.error("failed to load class", e)
+ case e:Exception => logger.error(s"failed to load class ${hookStr}")
}
}
hooks.toArray
@@ -117,20 +115,22 @@
else if(isSupportParallelism) whenAvailable(f) else ensureIdle(f)
ensureOp {
val engineExecutorContext = createEngineExecutorContext(executeRequest)
+ var hookedCode = executeRequest.code;
Utils.tryCatch{
enginePreExecuteHooks foreach {
hook => logger.info(s"${hook.hookName} begins to do a hook")
- hook.callPreExecuteHook(engineExecutorContext, executeRequest)
+ hookedCode = hook.callPreExecuteHook(engineExecutorContext, executeRequest, hookedCode)
logger.info(s"${hook.hookName} ends to do a hook")
}
}{
- case e:Exception => logger.info("failed to do with hook")
+ case e:Throwable => logger.info("failed to do with hook", e)
}
+ info(s"hooked after code:$hookedCode")
var response: ExecuteResponse = null
val incomplete = new StringBuilder
- val codes = Utils.tryCatch(codeParser.map(_.parse(executeRequest.code, engineExecutorContext)).getOrElse(Array(executeRequest.code))){
- e => warn("Your code failed to commit one line at a time, and is now ready to execute as a full commit(您的代码在进行一行一行代码提交时失败，现在准备按照全部提交的方式进行执行)",e)
- Array(executeRequest.code)
+ val codes = Utils.tryCatch(codeParser.map(_.parse(hookedCode, engineExecutorContext)).getOrElse(Array(hookedCode))){
+ e => info("Your code will be submitted in overall mode")
+ Array(hookedCode)
}
engineExecutorContext.setTotalParagraph(codes.length)
codes.indices.foreach { index =>
@@ -145,7 +145,7 @@
//engineExecutorContext.appendStdout(getName + ">> " + incomplete.toString().trim + " complete ")
response match {
case e: ErrorExecuteResponse =>
- error(s"execute code $code failed!", e.t)
+ error(s"execute code failed!", e.t)
return response
case SuccessExecuteResponse() =>
engineExecutorContext.appendStdout("\n")
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala
index 2e9a43c..f4e87dd 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala
@@ -23,11 +23,14 @@
import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSet, ResultSetWriter}
import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils
+import com.webank.wedatasphere.linkis.cs.storage.CSTableResultSetWriter
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration.{ENGINE_RESULT_SET_MAX_CACHE, ENGINE_RESULT_SET_STORE_PATH}
import com.webank.wedatasphere.linkis.engine.exception.EngineErrorException
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse}
+import com.webank.wedatasphere.linkis.storage.resultset.table.TableResultSet
import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter}
import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord}
import org.apache.commons.io.IOUtils
@@ -36,7 +39,7 @@
import scala.collection.mutable.ArrayBuffer
-class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{
+class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging {
private val resultSetFactory = ResultSetFactory.getInstance
private val resultSetWriters = ArrayBuffer[ResultSetWriter[_ <: MetaData, _ <: Record]]()
@@ -48,17 +51,21 @@
private val aliasNum = new AtomicInteger(0)
protected var storePath: Option[String] = None
- private val properties:java.util.Map[String,Object] = new util.HashMap[String, Object]()
+ private val properties: java.util.Map[String, Object] = new util.HashMap[String, Object]()
private var totalParagraph = 0
private var currentParagraph = 0
def kill(): Unit = interrupted = true
+
def isKilled: Boolean = interrupted
def getTotalParagraph: Int = totalParagraph
+
def setTotalParagraph(totalParagraph: Int): Unit = this.totalParagraph = totalParagraph
+
def getCurrentParagraph: Int = currentParagraph
+
def setCurrentParagraph(currentParagraph: Int): Unit = this.currentParagraph = currentParagraph
def pushProgress(progress: Float, progressInfo: Array[JobProgressInfo]): Unit =
@@ -66,10 +73,10 @@
def sendResultSet(resultSetWriter: ResultSetWriter[_ <: MetaData, _ <: Record]): Unit = {
val fileName = new File(resultSetWriter.toFSPath.getPath).getName
- val index = if(fileName.indexOf(".") < 0) fileName.length else fileName.indexOf(".")
- val alias = if(fileName.startsWith("_")) fileName.substring(1, index) else fileName.substring(0, fileName.indexOf("_"))
-// resultSetWriter.flush()
- Utils.tryFinally(sendResultSet(resultSetWriter.toString(), alias)){
+ val index = if (fileName.indexOf(".") < 0) fileName.length else fileName.indexOf(".")
+ val alias = if (fileName.startsWith("_")) fileName.substring(1, index) else fileName.substring(0, fileName.indexOf("_"))
+ // resultSetWriter.flush()
+ Utils.tryFinally(sendResultSet(resultSetWriter.toString(), alias)) {
IOUtils.closeQuietly(resultSetWriter)
resultSetWriters synchronized resultSetWriters -= resultSetWriter
}
@@ -78,8 +85,8 @@
def sendResultSet(output: String): Unit = sendResultSet(output, "_" + aliasNum.getAndIncrement())
def appendTextResultSet(output: String): Unit = {
- if(defaultResultSetWriter == null) aliasNum synchronized {
- if(defaultResultSetWriter == null) {
+ if (defaultResultSetWriter == null) aliasNum synchronized {
+ if (defaultResultSetWriter == null) {
defaultResultSetWriter = createDefaultResultSetWriter(ResultSetFactory.TEXT_TYPE)
defaultResultSetWriter.addMetaData(new LineMetaData())
resultSetWriters += defaultResultSetWriter
@@ -91,12 +98,12 @@
private def sendResultSet(output: String, alias: String): Unit = {
if (StringUtils.isEmpty(output)) return
if (resultSetFactory.isResultSetPath(output))
- engineExecutor.getResultSetListener.foreach{ l =>
+ engineExecutor.getResultSetListener.foreach { l =>
jobId.foreach(l.onResultSetCreated(_, output, alias))
resultSize += 1
}
else if (resultSetFactory.isResultSet(output))
- engineExecutor.getResultSetListener.foreach{ l =>
+ engineExecutor.getResultSetListener.foreach { l =>
jobId.foreach(l.onResultSetCreated(_, output, alias))
resultSize += 1
}
@@ -104,7 +111,9 @@
}
def setJobId(jobId: String) = this.jobId = Option(jobId)
+
def getJobId = jobId
+
def setStorePath(storePath: String) = this.storePath = Option(storePath)
def sendResultSet(outputExecuteResponse: OutputExecuteResponse): Unit = outputExecuteResponse match {
@@ -112,13 +121,13 @@
case output: OutputExecuteResponse => sendResultSet(output.getOutput, "_" + aliasNum.getAndIncrement())
}
- def getProperties:java.util.Map[String, Object] = properties
+ def getProperties: java.util.Map[String, Object] = properties
- def addProperty(key:String, value:String):Unit = properties.put(key, value)
+ def addProperty(key: String, value: String): Unit = properties.put(key, value)
protected def getDefaultStorePath: String = {
val path = ENGINE_RESULT_SET_STORE_PATH.getValue
- (if(path.endsWith("/")) path else path + "/") + "user" + "/" +
+ (if (path.endsWith("/")) path else path + "/") + "user" + "/" +
DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd") + "/" + Sender.getThisServiceInstance.getApplicationName +
"/" + System.nanoTime
}
@@ -139,16 +148,34 @@
def createResultSetWriter(resultSet: ResultSet[_ <: MetaData, _ <: Record], alias: String): ResultSetWriter[_ <: MetaData, _ <: Record] = {
val filePath = storePath.getOrElse(getDefaultStorePath)
- val fileName = if(StringUtils.isEmpty(alias)) "_" + aliasNum.getAndIncrement() else alias + "_" + aliasNum.getAndIncrement()
+ val fileName = if (StringUtils.isEmpty(alias)) "_" + aliasNum.getAndIncrement() else alias + "_" + aliasNum.getAndIncrement()
val resultSetPath = resultSet.getResultSetPath(new FsPath(filePath), fileName)
- val resultSetWriter = ResultSetWriter.getResultSetWriter(resultSet, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath)
+ //update by peaceWong 20200402
+ val resultSetWriter = resultSet match {
+ case result: TableResultSet =>
+ val contextIDStr = ContextServiceUtils.getContextIDStrByMap(getProperties)
+ val nodeName = ContextServiceUtils.getNodeNameStrByMap(getProperties)
+ if (StringUtils.isNotBlank(contextIDStr) && StringUtils.isNotBlank(nodeName)) {
+ new CSTableResultSetWriter(result, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, contextIDStr, nodeName, alias)
+ } else {
+ ResultSetWriter.getResultSetWriter(resultSet, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath)
+ }
+ case _ => ResultSetWriter.getResultSetWriter(resultSet, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath)
+ }
+ //update by peaceWong 20200402 end
resultSetWriters synchronized resultSetWriters += resultSetWriter
resultSetWriter
}
- def appendStdout(log: String): Unit = if(!engineExecutor.isEngineInitialized)
+ def appendStdout(log: String): Unit = if (!engineExecutor.isEngineInitialized)
engineExecutor.info(log) else engineExecutor.getLogListener.foreach(ll => jobId.foreach(ll.onLogUpdate(_, log)))
+ def sendProgress(progress: Float, progressInfos: Array[JobProgressInfo]): Unit = {
+ if (engineExecutor.isEngineInitialized) {
+ engineExecutor.getJobProgressListener.foreach(ll => jobId.foreach(ll.onProgressUpdate(_, progress, progressInfos)))
+ }
+ }
+
def close(): Unit = {
resultSetWriters.toArray.foreach(sendResultSet)
engineExecutor.getResultSetListener.foreach(l => jobId.foreach(l.onResultSizeCreated(_, resultSize)))
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala
index 38712a4..6dac410 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala
@@ -58,11 +58,12 @@
if(executor == null) synchronized {
if(executor == null) {
var options: JMap[String, String] = DWCArgumentsParser.getDWCOptionMap
+ //TODO getUDF peaceWong
getEngineHooks.foreach(hook => options = hook.beforeCreateEngine(options))
executor = getOrCreateEngineExecutorFactory().createExecutor(options)
+ executor.setCodeParser(getOrCreateCodeParser())
executor.init()
executor.setLogListener(jobLogListener)
- executor.setCodeParser(getOrCreateCodeParser())
executor.setResultSetListener(resultSetListener)
//TODO Consider adding timeout（考虑加上超时时间）
getEngineHooks.foreach(_.afterCreatedEngine(executor))
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala
index ac093c4..89f1a6f 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala
@@ -16,14 +16,17 @@
package com.webank.wedatasphere.linkis.engine.execute
+import com.webank.wedatasphere.linkis.engine.{PropertiesExecuteRequest, ResourceExecuteRequest}
import com.webank.wedatasphere.linkis.protocol.engine.RequestTask
import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteRequest, JobExecuteRequest, RunTypeExecuteRequest}
import com.webank.wedatasphere.linkis.scheduler.queue.{Job, JobInfo}
-
+import java.util
/**
* Created by enjoyyin on 2018/9/25.
*/
-abstract class EngineJob extends Job with SenderContainer
+abstract class EngineJob extends Job with SenderContainer{
+ val resourcesStr:String = "resources"
+}
class CommonEngineJob extends EngineJob with SyncSenderContainer {
protected var request: RequestTask = _
@@ -36,13 +39,24 @@
override def isJobSupportRetry: Boolean = false
override protected def jobToExecuteRequest: ExecuteRequest = {
+
if (request.getProperties.containsKey("runType") && request.getProperties.containsKey(RequestTask.RESULT_SET_STORE_PATH))
- return new ExecuteRequest with JobExecuteRequest with StorePathExecuteRequest with RunTypeExecuteRequest{
+ return new ExecuteRequest with JobExecuteRequest with StorePathExecuteRequest
+ with RunTypeExecuteRequest with ResourceExecuteRequest with PropertiesExecuteRequest {
override val code: String = request.getCode
override val jobId: String = CommonEngineJob.this.getId
override val storePath: String = request.getProperties.get(RequestTask.RESULT_SET_STORE_PATH).toString
- override val runType: String = request.getProperties.get("runType").toString
+ override val runType: String = if (request.getProperties.get("runType") != null) {
+ request.getProperties.get("runType").toString
+ } else "sql"
+ override def resources: util.List[Object] = properties.get(resourcesStr) match {
+ case rs:util.List[Object] => rs
+ case _ => logger.warn(s"${CommonEngineJob.this.getId} resources type is not correct")
+ null
+ }
+ override val properties: util.Map[String, Object] = request.getProperties
}
+
if(request.getProperties.containsKey(RequestTask.RESULT_SET_STORE_PATH)) new ExecuteRequest with JobExecuteRequest with StorePathExecuteRequest {
override val code: String = request.getCode
override val jobId: String = CommonEngineJob.this.getId
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala
index 9746275..9af88a8 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala
@@ -20,7 +20,6 @@
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook}
-import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest}
import com.webank.wedatasphere.linkis.server.JMap
import org.apache.commons.io.FileUtils
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala
index 6b4cfa7..10fb13a 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala
@@ -18,13 +18,12 @@
import java.util.concurrent.TimeUnit
-import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging, Utils}
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook}
import com.webank.wedatasphere.linkis.protocol.engine.EngineState.Idle
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState
import com.webank.wedatasphere.linkis.server.JMap
-import org.apache.commons.lang.StringUtils
/**
* Created by enjoyyin on 2018/9/27.
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala
index 4a1d2d2..2af1999 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala
@@ -21,7 +21,6 @@
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration._
import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook}
-import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest}
import com.webank.wedatasphere.linkis.server.JMap
@@ -30,10 +29,9 @@
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.io.FileUtils
import org.apache.commons.lang.StringUtils
-import org.codehaus.jackson.map.ObjectMapper
-import scala.collection.mutable
import scala.collection.JavaConversions._
+import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
abstract class UdfLoadEngineHook extends EngineHook with Logging{ self =>
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala
index 1bbbfcb..ae65605 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala
@@ -9,5 +9,5 @@
*/
trait EnginePreExecuteHook {
val hookName:String
- def callPreExecuteHook(engineExecutorContext:EngineExecutorContext, executeRequest: ExecuteRequest)
+ def callPreExecuteHook(engineExecutorContext:EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String
}
\ No newline at end of file
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala
index 8eba415..9550c42 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala
@@ -16,16 +16,12 @@
package com.webank.wedatasphere.linkis.engine.lock
-import java.util.concurrent.{ConcurrentHashMap, ScheduledThreadPoolExecutor, TimeUnit}
-
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.engine.LockManager
import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorManager
import com.webank.wedatasphere.linkis.scheduler.SchedulerContext
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState
-import scala.collection.JavaConversions._
-
class EngineTimedLockManager(schedulerContext: SchedulerContext) extends LockManager(schedulerContext) with Logging{
var executorLock: EngineTimedLock = null
diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala
index a4eae5a..5ba7c13 100644
--- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala
+++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala
@@ -36,7 +36,7 @@
def setLogListener(logListener: LogListener):Unit = this.logListener = logListener
def pushAllRemainLogs():Unit = {
- logger.info(s"start to push all remain logs, and size is ${logCache.getRemain.size()}")
+ logger.info(s"start to push all remain logs")
Thread.sleep(30)
//logCache.synchronized{
if (logListener == null){
diff --git a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala
index 1c83838..4ce6c62 100644
--- a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala
+++ b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala
@@ -16,11 +16,6 @@
package com.webank.wedatasphere.linkis.engine.execute
-import java.io.File
-
-import com.google.common.io.Resources
-import org.apache.commons.io.FileUtils
-
//object SparkCombinedCodeParserTest {
// def main(args: Array[String]): Unit = {
// val parser = new SparkCombinedCodeParser
diff --git a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala
index 5c5e371..893b5d4 100644
--- a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala
+++ b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.engine.lock
import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus
-import com.webank.wedatasphere.linkis.engine.lock.EngineTimedLockManagerTest.lockManager
import com.webank.wedatasphere.linkis.scheduler.SchedulerContext
import com.webank.wedatasphere.linkis.scheduler.event.{ScheduleEvent, SchedulerEventListener}
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState.ExecutorState
diff --git a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala
index 31a051c..d050ad7 100644
--- a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala
+++ b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala
@@ -16,7 +16,6 @@
package scala.com.webank.wedatasphere.linkis.engine.log
-import com.webank.wedatasphere.linkis.common.utils.Logging
import org.slf4j.{Logger, LoggerFactory}
/**
diff --git a/ujes/enginemanager/pom.xml b/ujes/enginemanager/pom.xml
index 196f1db..9843ac9 100644
--- a/ujes/enginemanager/pom.xml
+++ b/ujes/enginemanager/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-enginemanager</artifactId>
diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala
index 15af33f..fe8fca5 100644
--- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala
+++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala
@@ -22,7 +22,7 @@
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration
import com.webank.wedatasphere.linkis.enginemanager.exception.EngineManagerErrorException
-import com.webank.wedatasphere.linkis.enginemanager.impl.{UserEngineResource, UserTimeoutEngineResource}
+import com.webank.wedatasphere.linkis.enginemanager.impl.UserTimeoutEngineResource
import com.webank.wedatasphere.linkis.enginemanager.process.{CommonProcessEngine, ProcessEngine, ProcessEngineBuilder}
import com.webank.wedatasphere.linkis.protocol.engine.{EngineCallback, RequestEngine}
import com.webank.wedatasphere.linkis.rpc.Sender
diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala
index cc6a2b0..df2242f 100644
--- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala
+++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala
@@ -21,7 +21,7 @@
import com.webank.wedatasphere.linkis.common.log.LogUtils
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration
-import com.webank.wedatasphere.linkis.enginemanager.exception.{EngineManagerErrorException, EngineManagerWarnException}
+import com.webank.wedatasphere.linkis.enginemanager.exception.{EMRetryException, EngineManagerErrorException, EngineManagerWarnException}
import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine
import com.webank.wedatasphere.linkis.resourcemanager.{AvailableResource, NotEnoughResource, Resource}
@@ -49,7 +49,7 @@
val resource = Utils.tryThrow(getEngineManagerContext.getOrCreateEngineResourceFactory
.createEngineResource(realRequest)){t =>
warn(s"In the configuration of ${realRequest.creator}, there is a parameter configuration in the wrong format!(${realRequest.creator}的配置中，存在错误格式的参数配置！)", t)
- throw new EngineManagerErrorException(30000, s"In the configuration of ${realRequest.creator}, there is a parameter configuration in the wrong format!(${realRequest.creator}的配置中，存在错误格式的参数配置！)")
+ throw new EngineManagerErrorException(11011, s"In the configuration of ${realRequest.creator}, there is a parameter configuration in the wrong format!(${realRequest.creator}的配置中，存在错误格式的参数配置！)")
}
val nodeResourceInfo = this.registerResources()
val usedResource = getEngineManagerContext.getOrCreateEngineFactory.getUsedResources.getOrElse(Resource.getZeroResource(resource.getResource))
@@ -59,7 +59,7 @@
info("ProtectedResource: "+ nodeResourceInfo.protectedResource.toString)
info("UsedResource: "+ usedResource.toString)
info("RequestResource: "+ resource.getResource.toString)
- throw new EngineManagerErrorException(31000, "The remote server resource has been used up, please switch to the remote server and try again!(远程服务器资源已被用光，请切换远程服务器再试！)")
+ throw new EngineManagerWarnException(31000, "远程服务器资源已被用光，请切换远程服务器再试！")
}
getEngineManagerContext.getOrCreateResourceRequester.request(resource) match {
case NotEnoughResource(reason) =>
diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala
index e01d27d..2c16639 100644
--- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala
+++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala
@@ -17,7 +17,7 @@
package com.webank.wedatasphere.linkis.enginemanager.conf
import com.webank.wedatasphere.linkis.common.conf.{ByteType, CommonVars}
-import EngineManagerConfiguration.ENGINE_SPRING_APPLICATION_NAME
+import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration.ENGINE_SPRING_APPLICATION_NAME
import org.apache.commons.lang.time.DateFormatUtils
/**
diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/exception/EMRetryException.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/exception/EMRetryException.scala
new file mode 100644
index 0000000..ad30b46
--- /dev/null
+++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/exception/EMRetryException.scala
@@ -0,0 +1,10 @@
+package com.webank.wedatasphere.linkis.enginemanager.exception
+
+import com.webank.wedatasphere.linkis.common.exception.DWCRetryException
+
+/**
+ * created by cooperyang on 2019/12/11
+ * Description:
+ */
+class EMRetryException (errCode:Int, desc:String ) extends DWCRetryException(errCode,desc){
+}
diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala
index c09e944..e65c204 100644
--- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala
+++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala
@@ -56,12 +56,13 @@
}
protected def isJarExists(udfInfo: UDFInfo) : Boolean = {
- if(FileUtils.getFile(udfInfo.getPath).exists()){
- true
- } else {
- info(s"The jar file [${udfInfo.getPath}] of UDF [${udfInfo.getUdfName}] doesn't exist, ignore it.")
- false
- }
+ true
+// if(FileUtils.getFile(udfInfo.getPath).exists()){
+// true
+// } else {
+// info(s"The jar file [${udfInfo.getPath}] of UDF [${udfInfo.getUdfName}] doesn't exist, ignore it.")
+// false
+// }
}
protected def extractUdfInfos(requestEngine: RequestEngine): mutable.ArrayBuffer[UDFInfo] = {
diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala
index f751550..799203a 100644
--- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala
+++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.enginemanager.impl
-import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineManager, Engine, EngineManagerContext}
+import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineManager, EngineManagerContext}
import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo
import com.webank.wedatasphere.linkis.resourcemanager.service.annotation.{EnableResourceManager, RegisterResource}
import org.springframework.beans.factory.annotation.Autowired
diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala
index 583fcc4..599c221 100644
--- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala
+++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.enginemanager.impl
-import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils
import com.webank.wedatasphere.linkis.enginemanager._
import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration.ENGINE_SPRING_APPLICATION_NAME
import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration._
diff --git a/ujes/entrance/pom.xml b/ujes/entrance/pom.xml
index 575430f..596fe51 100644
--- a/ujes/entrance/pom.xml
+++ b/ujes/entrance/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-entrance</artifactId>
@@ -54,6 +54,19 @@
</dependency>
<dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cs-ujes-client</artifactId>
+ <version>${linkis.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
index 2cbe656..902733b 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java
@@ -22,7 +22,10 @@
import org.springframework.core.annotation.AliasFor;
import org.springframework.stereotype.Component;
-import java.lang.annotation.*;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
/**
* Created by enjoyyin on 2019/2/14.
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
index 62445f5..360e8ff 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java
@@ -90,18 +90,24 @@
@EntranceListenerBusBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceListenerBusBeanAnnotation.BEAN_NAME})
- public EntranceEventListenerBus<EntranceEventListener, EntranceEvent> generateEntranceEventListenerBus(){
+ public EntranceEventListenerBus<EntranceEventListener, EntranceEvent> generateEntranceEventListenerBus() {
EntranceEventListenerBus<EntranceEventListener, EntranceEvent> entranceEventListenerBus = new EntranceEventListenerBus<EntranceEventListener, EntranceEvent>();
entranceEventListenerBus.start();
return entranceEventListenerBus;
}
+ /**
+ * Update by peaceWong add CSEntranceInterceptor
+ *
+ * @return
+ */
@EntranceInterceptorBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceInterceptorBeanAnnotation.BEAN_NAME})
public EntranceInterceptor[] generateEntranceInterceptors() {
- return new EntranceInterceptor[] {new PythonCodeCheckInterceptor(), new DBInfoCompleteInterceptor(), new SparkCodeCheckInterceptor(),
+ return new EntranceInterceptor[]{new CSEntranceInterceptor(), new PythonCodeCheckInterceptor(), new DBInfoCompleteInterceptor(), new SparkCodeCheckInterceptor(),
new SQLCodeCheckInterceptor(), new VarSubstitutionInterceptor(), new LogPathCreateInterceptor(),
- new StorePathEntranceInterceptor(), new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(),new CommentInterceptor()};
+ new StorePathEntranceInterceptor(), new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(), new CommentInterceptor(),
+ };
}
@ErrorCodeListenerBeanAnnotation
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
index a66d74f..82da5b6 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java
@@ -16,9 +16,7 @@
package com.webank.wedatasphere.linkis.entrance.exception;
-import com.webank.wedatasphere.linkis.common.exception.DWCRuntimeException;
import com.webank.wedatasphere.linkis.common.exception.ErrorException;
-import com.webank.wedatasphere.linkis.common.exception.ExceptionLevel;
/**
* created by enjoyyin on 2018/10/8
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
index 2d600b0..1f5585b 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java
@@ -16,7 +16,6 @@
package com.webank.wedatasphere.linkis.entrance.job;
-import com.webank.wedatasphere.linkis.common.conf.TimeType;
import com.webank.wedatasphere.linkis.common.log.LogUtils;
import com.webank.wedatasphere.linkis.common.utils.Utils;
import com.webank.wedatasphere.linkis.entrance.execute.*;
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
index 5b0d1a3..78a2777 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java
@@ -112,6 +112,7 @@
//job.setProgressListener(entranceContext.getOrCreatePersistenceManager());
//job.setJobListener(entranceContext.getOrCreatePersistenceManager());
job.setEntranceListenerBus(entranceContext.getOrCreateEventListenerBus());
+ job.setEntranceContext(entranceContext);
job.setListenerEventBus(null);
job.setProgress(0f);
}
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
index 7088234..67daab2 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java
@@ -23,6 +23,7 @@
package com.webank.wedatasphere.linkis.entrance.persistence;
import com.google.gson.Gson;
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration;
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration$;
import com.webank.wedatasphere.linkis.entrance.exception.EntranceIllegalParamException;
import com.webank.wedatasphere.linkis.entrance.exception.EntranceRPCException;
@@ -82,10 +83,6 @@
if (object == null){
throw new QueryFailedException(20011, "insert task failed, reason: " + message);
}
-/* if (object instanceof Long){
- Long taskID = (Long)object;
- ((RequestPersistTask) task).setTaskID(taskID);
- }*/
String taskStr = object.toString();
Long taskID = Long.parseLong(taskStr.substring(0,taskStr.indexOf(".")));
((RequestPersistTask) task).setTaskID(taskID);
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
index 9a89ff8..27edd6a 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java
@@ -25,6 +25,7 @@
import com.webank.wedatasphere.linkis.common.exception.ErrorException;
import com.webank.wedatasphere.linkis.common.io.FsPath;
import com.webank.wedatasphere.linkis.entrance.EntranceContext;
+import com.webank.wedatasphere.linkis.entrance.cs.CSEntranceHelper;
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob;
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo;
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask;
@@ -79,7 +80,6 @@
public void onResultSetCreated(Job job, OutputExecuteResponse response) {
String path;
boolean isEntranceJob = job instanceof EntranceJob;
-// if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersist();
try {
path = createResultSetEngine().persistResultSet(job, response);
} catch (Throwable e) {
@@ -98,14 +98,18 @@
} catch (Throwable e1){
logger.error("job {} onLogUpdate error, reason:", job.getId(), e1);
} //ignore it
- if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersisted();
+ if(isEntranceJob) {
+ ((EntranceJob)job).incrementResultSetPersisted();
+ }
return;
}
if(task instanceof RequestPersistTask) {
RequestPersistTask requestPersistTask = (RequestPersistTask) task;
if(StringUtils.isEmpty(requestPersistTask.getResultLocation())) synchronized (task) {
if(StringUtils.isNotEmpty(requestPersistTask.getResultLocation())) {
- if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersisted();
+ if(isEntranceJob) {
+ ((EntranceJob)job).incrementResultSetPersisted();
+ }
return;
}
try {
@@ -117,7 +121,9 @@
}
}
}
- if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersisted();
+ if(isEntranceJob) {
+ ((EntranceJob)job).incrementResultSetPersisted();
+ }
}
@Override
@@ -153,6 +159,15 @@
@Override
public void onJobCompleted(Job job) {
+ //update by peaceWong(2020/05/10) to set jobID to CS
+ try {
+ if (job.isSucceed()) {
+ CSEntranceHelper.registerCSRSData(job);
+ }
+ } catch (Throwable e) {
+ logger.error("Failed to register cs rs data ", e);
+ }
+ //end update
updateJobStatus(job);
}
@@ -163,12 +178,15 @@
}
try{
task = this.entranceContext.getOrCreateEntranceParser().parseToTask(job);
+ if (job.isSucceed()){
+ //如果是job是成功的，那么需要将task的错误描述等都要设置为null
+ ((RequestPersistTask)task).setErrCode(null);
+ ((RequestPersistTask)task).setErrDesc(null);
+ }
}catch(ErrorException e){
entranceContext.getOrCreateLogManager().onLogUpdate(job, e.getMessage());
logger.error("update job status failed, reason:", e);
}
- //TODO If the execution fails, there may be an error message, you need to call job.getErrorResponse to persist the error message.(如果是执行失败了，可能会有错误信息，需要调用job.getErrorResponse持久化错误信息)
- //TODO The error message is compared with the error code of errorListener(错误信息要跟errorListener的错误码信息对比)
try {
createPersistenceEngine().updateIfNeeded(task);
} catch (ErrorException e) {
diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java
index 2a23bb5..44238f8 100644
--- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java
+++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java
@@ -20,9 +20,9 @@
import com.webank.wedatasphere.linkis.entrance.EntranceServer;
import com.webank.wedatasphere.linkis.entrance.annotation.EntranceServerBeanAnnotation;
import com.webank.wedatasphere.linkis.entrance.background.BackGroundService;
-import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration;
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob;
import com.webank.wedatasphere.linkis.entrance.log.LogReader;
+import com.webank.wedatasphere.linkis.entrance.utils.JobHistoryHelper;
import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant;
import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo;
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask;
@@ -37,9 +37,7 @@
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import org.springframework.web.bind.annotation.RequestParam;
import scala.Option;
import javax.servlet.http.HttpServletRequest;
@@ -113,10 +111,22 @@
@Override
@GET
@Path("/{id}/status")
- public Response status(@PathParam("id") String id) {
+ public Response status(@PathParam("id") String id, @QueryParam("taskID")String taskID) {
Message message = null;
String realId = ZuulEntranceUtils.parseExecID(id)[3];
- Option<Job> job = entranceServer.getJob(realId);
+ Option<Job> job = Option.apply(null);
+ try{
+ job = entranceServer.getJob(realId);
+ }catch(Exception e){
+ logger.warn("获取任务 {} 状态时出现错误", realId, e);
+ //如果获取错误了,证明在内存中已经没有了,去jobhistory找寻一下taskID代表的任务的状态，然后返回
+ long realTaskID = Long.parseLong(taskID);
+ String status = JobHistoryHelper.getStatusByTaskID(realTaskID);
+ message = Message.ok();
+ message.setMethod("/api/entrance/" + id + "/status");
+ message.data("status",status).data("execID", id);
+ return Message.messageToResponse(message);
+ }
if (job.isDefined()){
message = Message.ok();
message.setMethod("/api/entrance/" + id + "/status");
@@ -139,20 +149,20 @@
Option<Job> job = entranceServer.getJob(realId);
if (job.isDefined()){
JobProgressInfo[] jobProgressInfos = ((EntranceJob)job.get()).getProgressInfo();
- Map<String, Object> map = new HashMap<>();
if (jobProgressInfos == null){
message = Message.error("Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)");
message.setMethod("/api/entrance/" + id + "/progress");
}else{
- for(JobProgressInfo jobProgressInfo : jobProgressInfos){
+ List<Map<String, Object>> list = new ArrayList<>();
+ for(JobProgressInfo jobProgressInfo : jobProgressInfos) {
+ Map<String, Object> map = new HashMap<>();
map.put("id", jobProgressInfo.id());
map.put("succeedTasks", jobProgressInfo.succeedTasks());
map.put("failedTasks", jobProgressInfo.failedTasks());
map.put("runningTasks", jobProgressInfo.runningTasks());
map.put("totalTasks", jobProgressInfo.totalTasks());
+ list.add(map);
}
- List<Map<String, Object>> list = new ArrayList<>();
- list.add(map);
message = Message.ok();
message.setMethod("/api/entrance/" + id + "/progress");
message.data("progress",job.get().getProgress()).data("execID", id).data("progressInfo", list);
@@ -279,9 +289,21 @@
@Override
@GET
@Path("/{id}/kill")
- public Response kill(@PathParam("id")String id) {
+ public Response kill(@PathParam("id")String id, @QueryParam("taskID") long taskID) {
String realId = ZuulEntranceUtils.parseExecID(id)[3];
- Option<Job> job = entranceServer.getJob(realId);
+ //通过jobid获取job,可能会由于job找不到而导致有looparray的报错,一旦报错的话，就可以将该任务直接置为Cancenlled
+ Option<Job> job = Option.apply(null);
+ try{
+ job = entranceServer.getJob(realId);
+ }catch(Exception e){
+ logger.warn("can not find a job in entranceServer, will force to kill it", e);
+ //如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的
+ JobHistoryHelper.forceKill(taskID);
+ Message message = Message.ok("强制杀死任务");
+ message.setMethod("/api/entrance/" + id + "/kill");
+ message.setStatus(0);
+ return Message.messageToResponse(message);
+ }
Message message = null;
if (job.isEmpty()){
message = Message.error("Can't find execID(不能找到execID): " + id + "Corresponding job, can't kill(对应的job，不能进行kill)");
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
index faec2d8..da2188f 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala
@@ -18,7 +18,6 @@
import com.webank.wedatasphere.linkis.common.exception.{DWCException, DWCRuntimeException, ErrorException}
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
-import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException}
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
import com.webank.wedatasphere.linkis.entrance.log.LogReader
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
index 49f26c8..89ae610 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala
@@ -309,7 +309,7 @@
if (StringUtils.isBlank(log)) return
var message:Message = null
val logs:Array[String] = new Array[String](4)
- val logArr:Array[String] = log.split("\n")
+ val logArr:Array[String] = log.split("\n\n").filter(StringUtils.isNotBlank)
val info = new StringBuilder
val warn = new StringBuilder
val error = new StringBuilder
@@ -321,14 +321,29 @@
case ERROR_HEADER1() | ERROR_HEADER2() =>
concatLog(length, singleLog, error, all)
case WARN_HEADER1() | WARN_HEADER2() =>
- val arr = EntranceConfiguration.LOG_EXCLUDE.getValue.split(",").map (word => word.trim)
+ val arr = EntranceConfiguration.LOG_WARN_EXCLUDE.getValue.split(",").map (word => word.trim)
var flag = false
for (keyword <- arr){
flag = singleLog.contains(keyword) || flag
}
- if (!flag) concatLog(length, singleLog, warn, all)
+ if (!flag) {
+ val message = singleLog.split("\n")(0)
+ concatLog(length, message, warn, all)
+ }
case INFO_HEADER1() | INFO_HEADER2() =>
val hiveLogSpecial:String = EntranceConfiguration.HIVE_SPECIAL_LOG_INCLUDE.getValue
+ val sparkLogSpecial:String = EntranceConfiguration.SPARK_SPECIAL_LOG_INCLUDE.getValue
+ val hiveCreateTableLog:String = EntranceConfiguration.HIVE_CREATE_TABLE_LOG.getValue
+ if (singleLog.contains(hiveLogSpecial) && singleLog.contains(hiveCreateTableLog)){
+ val threadName = EntranceConfiguration.HIVE_THREAD_NAME.getValue
+ val printInfo = EntranceConfiguration.HIVE_PRINT_INFO_LOG.getValue
+ val start = singleLog.indexOf(threadName)
+ val end = singleLog.indexOf(printInfo) + printInfo.length
+ if(start > 0 && end > 0) {
+ val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length)
+ concatLog(length, realLog, info, all)
+ }
+ }
if (singleLog.contains(hiveLogSpecial) && singleLog.contains("map") && singleLog.contains("reduce")){
val threadName = EntranceConfiguration.HIVE_THREAD_NAME.getValue
val stageName = EntranceConfiguration.HIVE_STAGE_NAME.getValue
@@ -338,6 +353,15 @@
val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length)
concatLog(length, realLog, info, all)
}
+ }else if (singleLog.contains(sparkLogSpecial)){
+ val className = EntranceConfiguration.SPARK_PROGRESS_NAME.getValue
+ val endFlag = EntranceConfiguration.END_FLAG.getValue
+ val start = singleLog.indexOf(className)
+ val end = singleLog.indexOf(endFlag) + endFlag.length
+ if(start > 0 && end > 0) {
+ val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length)
+ concatLog(length, realLog, info, all)
+ }
}else{
val arr = EntranceConfiguration.LOG_EXCLUDE.getValue.split(",").map (word => word.trim)
var flag = false
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala
index 9fc0aa2..feb665a 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala
@@ -85,6 +85,7 @@
*/
val DEFAULT_RUN_TYPE = CommonVars("wds.linkis.default.runType", "sql")
+ val LOG_WARN_EXCLUDE = CommonVars("wds.linkis.warn.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,com.webank.wedatasphere")
val CLEAR_LOG = CommonVars("wds.linkis.log.clear", false)
@@ -92,7 +93,7 @@
* LOG_EXCLUDE is used to remove the log of the framework log, such as hive spark spring, so that it is not pushed to the front end through websocket.
* LOG_EXCLUDE 是用来进行把框架日志，比如hive spark spring等日志进行剔除，不让其通过websocket进行推送到前端
*/
- val LOG_EXCLUDE = CommonVars("wds.linkis.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,cn.webank.bdp,com.webank")
+ val LOG_EXCLUDE = CommonVars("wds.linkis.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,com.webank.wedatasphere,com.webank")
/**
* wds.linkis.dwc.instance is a parameter used to control the number of engines each user starts.
@@ -119,4 +120,26 @@
val HIVE_STAGE_NAME = CommonVars("wds.linkis.hive.stage.name", "Stage-")
+ val SPARK_SPECIAL_LOG_INCLUDE = CommonVars("wds.linkis.spark.special.log.include", "com.webank.wedatasphere.linkis.engine.spark.utils.JobProgressUtil")
+
+
+ val SPARK_PROGRESS_NAME = CommonVars("wds.linkis.spark.progress.name", "com.webank.wedatasphere.linkis.engine.spark.utils.JobProgressUtil$")
+
+ val END_FLAG = CommonVars("bdp.dataworkcloud.entrance.end.flag", "info -")
+
+ val HIVE_CREATE_TABLE_LOG = CommonVars("wds.linkis.hive.create.table.log", "numFiles")
+
+ val HIVE_PRINT_INFO_LOG = CommonVars("wds.linkis.hive.printinfo.log", "printInfo -")
+
+
+
+ val IS_BDP_ENV = CommonVars("wds.linkis.entrance.bdp.env", "true")
+
+
+ val SHELL_DANGER_USAGE = CommonVars("wds.linkis.shell.danger.usage", "rm,sh,find,kill,python,for,source,hdfs,hadoop,spark-sql,spark-submit,pyspark,spark-shell,hive,yarn")
+ val SHELL_WHITE_USAGE = CommonVars("wds.linkis.shell.white.usage", "cd,ls")
+
+ val FLOW_EXECUTION_CREATOR = CommonVars("wds.linkis.entrance.flow.creator", "nodeexecution")
+
+ val SCHEDULER_CREATOR = CommonVars("wds.linkis.entrance.scheduler.creator", "scheduler")
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala
new file mode 100644
index 0000000..1caa132
--- /dev/null
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.cs
+
+import java.util
+
+import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.cs.client.service.{CSNodeServiceImpl, CSVariableService, LinkisJobDataServiceImpl}
+import com.webank.wedatasphere.linkis.cs.client.utils.{ContextServiceUtils, SerializeHelper}
+import com.webank.wedatasphere.linkis.cs.common.entity.`object`.LinkisVariable
+import com.webank.wedatasphere.linkis.cs.common.entity.data.LinkisJobData
+import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.{ContextScope, ContextType}
+import com.webank.wedatasphere.linkis.cs.common.entity.source.{CommonContextKey, LinkisWorkflowContextID}
+import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
+import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant
+import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils
+import com.webank.wedatasphere.linkis.scheduler.queue.Job
+import org.apache.commons.lang.StringUtils
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable
+
+/**
+ * @author peacewong
+ * @date 2020/3/5 15:33
+ */
+object CSEntranceHelper extends Logging {
+
+
+ def getContextInfo(params: util.Map[String, Any]): (String, String) = {
+
+ val runtimeMap = params.get(TaskConstant.PARAMS_CONFIGURATION) match {
+ case map: util.Map[String, AnyRef] => map.get(TaskConstant.PARAMS_CONFIGURATION_RUNTIME)
+ case _ => null
+ }
+
+ if (null != runtimeMap) {
+ runtimeMap match {
+ case map: util.Map[String, AnyRef] =>
+ val name = ContextServiceUtils.getNodeNameStrByMap(map)
+ return (ContextServiceUtils.getContextIDStrByMap(map), name)
+ case _ =>
+ }
+ }
+ (null, null)
+ }
+
+ def setContextInfo(params: util.Map[String, Any], copyMap: util.Map[String, String]): Unit = {
+ val (contextIDValueStr, nodeNameStr) = getContextInfo(params)
+ if (StringUtils.isNotBlank(contextIDValueStr)) {
+ copyMap.put(CSCommonUtils.CONTEXT_ID_STR, contextIDValueStr)
+ copyMap.put(CSCommonUtils.NODE_NAME_STR, nodeNameStr)
+ }
+ }
+
+
+ /**
+ * register job id to cs
+ *
+ * @param job
+ */
+ def registerCSRSData(job: Job): Unit = {
+ job match {
+ case entranceJob: EntranceJob => {
+ val (contextIDValueStr, nodeNameStr) = getContextInfo(entranceJob.getParams)
+ info(s"registerCSRSData: nodeName:$nodeNameStr")
+ if (StringUtils.isBlank(contextIDValueStr) || StringUtils.isBlank(nodeNameStr)) return null
+
+ val contextKey = new CommonContextKey
+ contextKey.setContextScope(ContextScope.PUBLIC)
+ contextKey.setContextType(ContextType.DATA)
+ contextKey.setKey(CSCommonUtils.NODE_PREFIX + nodeNameStr + CSCommonUtils.JOB_ID)
+ entranceJob.getTask match {
+ case requestPersistTask: RequestPersistTask =>
+ val data = new LinkisJobData
+ data.setJobID(requestPersistTask.getTaskID)
+ LinkisJobDataServiceImpl.getInstance().putLinkisJobData(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey), data)
+ info(s"(${contextKey.getKey} put ${requestPersistTask.getTaskID} of taskID to cs)")
+ case _ =>
+ }
+ info(s"registerCSRSData end: nodeName:$nodeNameStr")
+ }
+ case _ =>
+ }
+ }
+
+ /**
+ * initNodeCSInfo
+ *
+ * @param requestPersistTask
+ * @return
+ */
+ def initNodeCSInfo(requestPersistTask: RequestPersistTask): Unit = {
+
+ val (contextIDValueStr, nodeNameStr) = getContextInfo(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]])
+
+ if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) {
+ info(s"init node($nodeNameStr) cs info")
+ CSNodeServiceImpl.getInstance().initNodeCSInfo(contextIDValueStr, nodeNameStr)
+ }
+ }
+
+
+ /**
+ * reset creator by contextID information
+ * 1. Not set If contextID does not exists
+ * 2. If env of contextID are dev set nodeexecution
+ * 3. If env of contextID are prod set scheduler
+ *
+ * @param requestPersistTask
+ */
+ def resetCreator(requestPersistTask: RequestPersistTask): Unit = {
+
+ val (contextIDValueStr, nodeNameStr) = getContextInfo(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]])
+
+ if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) {
+ SerializeHelper.deserializeContextID(contextIDValueStr) match {
+ case contextID: LinkisWorkflowContextID =>
+ if (CSCommonUtils.CONTEXT_ENV_PROD.equalsIgnoreCase(contextID.getEnv)) {
+ info(s"reset creator from ${requestPersistTask.getRequestApplicationName} to " + EntranceConfiguration.SCHEDULER_CREATOR.getValue)
+ requestPersistTask.setRequestApplicationName(EntranceConfiguration.SCHEDULER_CREATOR.getValue)
+ } else {
+ info(s"reset creator from ${requestPersistTask.getRequestApplicationName} to " + EntranceConfiguration.FLOW_EXECUTION_CREATOR.getValue)
+ requestPersistTask.setRequestApplicationName(EntranceConfiguration.FLOW_EXECUTION_CREATOR.getValue)
+ }
+ case _ =>
+ }
+ }
+ }
+
+
+ /**
+ * From cs to get variable
+ *
+ * @param requestPersistTask
+ * @return
+ */
+ def addCSVariable(requestPersistTask: RequestPersistTask): Unit = {
+ val variableMap = new mutable.HashMap[String, String]()
+ val (contextIDValueStr, nodeNameStr) = getContextInfo(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]])
+
+ if (StringUtils.isNotBlank(contextIDValueStr)) {
+ info(s"parse variable nodeName:$nodeNameStr")
+ val linkisVariableList: util.List[LinkisVariable] = CSVariableService.getInstance().getUpstreamVariables(contextIDValueStr, nodeNameStr);
+ if (null != linkisVariableList) {
+ linkisVariableList.foreach { linkisVariable =>
+ variableMap.put(linkisVariable.getKey, linkisVariable.getValue)
+ }
+ }
+ TaskUtils.addVariableMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]], variableMap)
+ info(s"parse variable end nodeName:$nodeNameStr")
+ }
+ }
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala
index 4a9e8b1..29ef5f0 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala
@@ -22,7 +22,7 @@
*/
package com.webank.wedatasphere.linkis.entrance.exception
-import com.webank.wedatasphere.linkis.common.exception.{DWCException, ErrorException, ExceptionLevel}
+import com.webank.wedatasphere.linkis.common.exception.{ErrorException, ExceptionLevel}
case class CacheNotReadyException(errCode:Int,
desc:String) extends ErrorException(errCode, desc){
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala
new file mode 100644
index 0000000..132a561
--- /dev/null
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala
@@ -0,0 +1,9 @@
+package com.webank.wedatasphere.linkis.entrance.exception
+
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+
+/**
+ * created by cooperyang on 2020/1/2
+ * Description:
+ */
+case class JobHistoryFailedException(errorMsg:String) extends ErrorException(50081, errorMsg)
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala
index 40fe6db..f3ea61c 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala
@@ -16,8 +16,12 @@
package com.webank.wedatasphere.linkis.entrance.execute
+import java.util.Date
+
import com.webank.wedatasphere.linkis.common.exception.WarnException
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob
+import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState.ExecutorState
import com.webank.wedatasphere.linkis.scheduler.executer.{Executor, ExecutorManager}
import com.webank.wedatasphere.linkis.scheduler.listener.ExecutorListener
@@ -77,11 +81,8 @@
protected def findExecutors(job: Job): Array[EntranceEngine] = {
val groupName = groupFactory.getGroupNameByEvent(job)
- //logger.info(s"${job.getId} groupName is ${groupName}")
var engines = getOrCreateEngineManager().listEngines(_.getGroup.getGroupName == groupName)
- //logger.info(s"in findExecutors fun engines is $engines")
getOrCreateEntranceExecutorRulers().foreach(ruler => engines = ruler.rule(engines, job))
- //logger.info(s"after findExecutors fun engines is $engines")
engines
}
@@ -111,6 +112,11 @@
findUsefulExecutor(job).orElse {
val executor = createExecutor(job)
if(executor != null) {
+ job match{
+ case entranceExecutionJob: EntranceExecutionJob => val task = entranceExecutionJob.getTask
+ task.asInstanceOf[RequestPersistTask].setEngineStartTime(new Date())
+ case _ =>
+ }
if(!job.isCompleted){
val lock = getOrCreateEngineSelector().lockEngine(executor)
setLock(lock, job)
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala
index bb49a1e..45ae26e 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala
@@ -20,6 +20,7 @@
import java.util.concurrent.atomic.AtomicInteger
import com.webank.wedatasphere.linkis.common.log.LogUtils
+import com.webank.wedatasphere.linkis.entrance.EntranceContext
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.event._
import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException
@@ -47,6 +48,7 @@
private var progressInfo:Array[JobProgressInfo] = Array.empty
private val persistedResultSets = new AtomicInteger(0)
private var resultSize = -1
+ private var entranceContext:EntranceContext = _
def getTask:Task = task
def setTask(task:Task):Unit = this.task = task
def setCreator(creator: String): Unit = this.creator = creator
@@ -60,6 +62,9 @@
def getEntranceListenerBus = this.entranceListenerBus
def setProgressInfo(progressInfo:Array[JobProgressInfo]):Unit = this.progressInfo = progressInfo
def getProgressInfo:Array[JobProgressInfo] = this.progressInfo
+ def setEntranceContext(entranceContext: EntranceContext):Unit = this.entranceContext = entranceContext
+ def getEntraceCotnext:EntranceContext = this.entranceContext
+
def setResultSize(resultSize: Int): Unit = {
this.resultSize = resultSize
@@ -138,6 +143,11 @@
super.transitionCompleted(executeCompleted)
}
+ def transitionCompleted(executeCompleted: CompletedExecuteResponse, reason: String): Unit = {
+ info("Job directly completed with reason: " + reason)
+ transitionCompleted(executeCompleted)
+ }
+
override protected def isJobShouldRetry(errorExecuteResponse: ErrorExecuteResponse): Boolean = isJobSupportRetry && errorExecuteResponse != null &&
(if(RPCUtils.isReceiverNotExists(errorExecuteResponse.t)) {
getExecutor match {
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala
index 5803fbc..99e9ea0 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala
@@ -19,10 +19,9 @@
import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.EntranceContext
import com.webank.wedatasphere.linkis.entrance.annotation.EntranceContextBeanAnnotation
-import com.webank.wedatasphere.linkis.entrance.event.{EntranceProgressEvent}
+import com.webank.wedatasphere.linkis.entrance.event.EntranceProgressEvent
import com.webank.wedatasphere.linkis.entrance.utils.RPCUtils
import com.webank.wedatasphere.linkis.protocol.engine._
-
import com.webank.wedatasphere.linkis.rpc.exception.DWCRPCRetryException
import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender}
import com.webank.wedatasphere.linkis.scheduler.executer.AliasOutputExecuteResponse
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
index c15e618..0d88e5a 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.entrance.interceptor
-import com.webank.wedatasphere.linkis.common.exception.{ErrorException, WarnException}
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.protocol.task.Task
/**
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
index 9274252..b371fda 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala
@@ -16,7 +16,7 @@
package com.webank.wedatasphere.linkis.entrance.interceptor.exception
-import com.webank.wedatasphere.linkis.common.exception.{ErrorException, WarnException}
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
/**
* created by enjoyyin on 2018/10/19
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala
new file mode 100644
index 0000000..26629f8
--- /dev/null
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.webank.wedatasphere.linkis.entrance.interceptor.impl
+
+import java.lang
+
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.cs.CSEntranceHelper
+import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
+import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
+import com.webank.wedatasphere.linkis.protocol.task.Task
+
+/**
+ * @author peacewong
+ * @date 2020/3/24 18:28
+ */
+class CSEntranceInterceptor extends EntranceInterceptor with Logging {
+
+ override def apply(task: Task, logAppender: lang.StringBuilder): Task = {
+ task match {
+ case requestPersistTask: RequestPersistTask =>
+ logger.info("Start to execute CSEntranceInterceptor")
+ Utils.tryAndWarn(CSEntranceHelper.addCSVariable(requestPersistTask))
+ Utils.tryAndWarn(CSEntranceHelper.resetCreator(requestPersistTask))
+ Utils.tryAndWarn(CSEntranceHelper.initNodeCSInfo(requestPersistTask))
+ logger.info("Finished to execute CSEntranceInterceptor")
+ case _ =>
+ }
+ task
+ }
+}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala
index 0fa8c97..4d34ed4 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala
@@ -22,9 +22,8 @@
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
-import org.apache.commons.lang.StringUtils
+import org.slf4j.{Logger, LoggerFactory}
-import scala.collection.mutable.ArrayBuffer
import scala.util.matching.Regex
/**
@@ -63,26 +62,18 @@
object SQLCommentHelper extends CommentHelper {
override val commentPattern: Regex = """\s*--.+\s*""".r.unanchored
private val comment = "(?ms)([\"']*['\"])|--.*?$|/\\*.*?\\*/"
+ private val logger:Logger = LoggerFactory.getLogger(getClass)
override def dealComment(code: String): String = {
- val p = Pattern.compile(comment)
- val sql = p.matcher(code).replaceAll("$1")
- sql
-// val clearCode = new ArrayBuffer[String]()
-//// code.split(";") foreach {
-//// case commentPattern() =>
-//// case singleCode:String => clearCode += singleCode
-//// case _ =>
-//// }
-//// clearCode.mkString(";")
-// code.split(";") foreach(singleLine =>{
-// val tempCode = new ArrayBuffer[String]()
-// singleLine.split("\n") foreach (line => tempCode +=
-// line.substring(0, line.length - commentPattern.findFirstIn(line).getOrElse("").length))
-// clearCode += tempCode.mkString("\n")
-// })
-// val resultCode = new ArrayBuffer[String]()
-// clearCode.filter(StringUtils.isNotBlank).foreach(resultCode += _)
-// resultCode.mkString(";")
+ try{
+ val p = Pattern.compile(comment)
+ val sql = p.matcher(code).replaceAll("$1")
+ sql
+ }catch{
+ case e:Exception => logger.warn("sql comment failed")
+ code
+ case t:Throwable => logger.warn("sql comment failed")
+ code
+ }
}
}
@@ -90,16 +81,7 @@
override val commentPattern: Regex = """^\s*#.+\s*""".r.unanchored
val pythonCommentPattern:String = "(?ms)([\"'](?:|[^'])*['\"])|#.*?$|/\\*.*?\\*/"
override def dealComment(code: String): String = {
- //val p = Pattern.compile(pythonCommentPattern)
- //p.matcher(code).replaceAll("$1")
code
-// val clearCode = new ArrayBuffer[String]()
-// code.split("\n") foreach {
-// case commentPattern() =>
-// case singleCode:String => clearCode += singleCode
-// case _ =>
-// }
-// clearCode.mkString("\n")
}
}
@@ -110,13 +92,6 @@
override def dealComment(code: String): String = {
val p = Pattern.compile(scalaCommentPattern)
p.matcher(code).replaceAll("$1")
-// val clearCode = new ArrayBuffer[String]()
-// code.split("\n") foreach {
-// case commentPattern() =>
-// case singleCode:String => clearCode += singleCode
-// case _ =>
-// }
-// clearCode.mkString("\n")
}
}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
index 99b6e98..7350f93 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala
@@ -21,7 +21,6 @@
import java.util
import java.util.{Calendar, Date}
-//import com.sun.jdi.FloatValue
import com.webank.wedatasphere.linkis.common.utils.Logging
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.VarSubstitutionException
@@ -31,13 +30,11 @@
import com.webank.wedatasphere.linkis.protocol.variable.{RequestQueryAppVariable, ResponseQueryVariable}
import com.webank.wedatasphere.linkis.rpc.Sender
import org.apache.commons.lang.StringUtils
-
-import scala.collection.mutable.ArrayBuffer
-//import com.webank.wedatasphere.linkis.entrance.log.LogManager
import org.apache.commons.lang.time.DateUtils
import scala.collection.JavaConversions._
import scala.collection.mutable
+import scala.collection.mutable.ArrayBuffer
import scala.util.control.Exception._
/**
@@ -66,8 +63,8 @@
* 1. 从代码中得到用户定义的变量，进行替换
* 2. 如果1没有做，那么从args中得到用户定义的变量，进行替换
- * 3. 如果2没有做，从控制台中得到用户定义的变量，进行替换
- *
+ * 3. 如果2没有做，从CS中得到用户定义的变量，进行替换
+ *3. 如果3没有做，从控制台中得到用户定义的变量，进行替换
* @param task : requestPersistTask
* @return
*/
@@ -77,7 +74,7 @@
var codeType = SQL_TYPE
runType match {
case "hql" | "sql" | "jdbc" | "hive" => codeType = SQL_TYPE
- case "python" => codeType = PY_TYPE
+ case "python" | "py" => codeType = PY_TYPE
case "java" => codeType = JAVA_TYPE
case "scala" => codeType = SCALA_TYPE
case _ => return (false, code)
@@ -86,106 +83,57 @@
var run_date:CustomDateType = null
val nameAndType = mutable.Map[String, VariableType]()
val nameAndValue: mutable.Map[String, String] = getCustomVar(code, codeType)
- /* The first step is to replace the variable from code*/
- /*第一步来自code的变量替换*/
- nameAndValue.foreach {
- case (name, value) => {
- name match {
- case RUN_DATE => {
- if (value.length == 8) run_date = new CustomDateType(value, false)
- else
- throw VarSubstitutionException(20040, "please use correct date format,example:run_date=20170101")
- }
- case _ => /*if ((allCatch opt value.toLong).isDefined) {
- nameAndType(name) = LongType(value.toLong)
- } else*/ if ((allCatch opt value.toDouble).isDefined) {
- nameAndType(name) = DoubleValue(value.toDouble)
- } else {
- nameAndType(name) = StringType(value)
+
+ def putNameAndType(data: mutable.Map[String, String]): Unit = if (null != data) data foreach {
+ case (key, value) => key match {
+ case RUN_DATE => if (nameAndType.get(RUN_DATE).isEmpty) {
+ val run_date_str = value.asInstanceOf[String]
+ if (StringUtils.isNotEmpty(run_date_str)) {
+ run_date = new CustomDateType(run_date_str, false)
+ nameAndType(RUN_DATE) = DateType(run_date)
}
}
- }
- }
- if (run_date != null){
- nameAndType(RUN_DATE) = DateType(run_date)
- }
- /* Perform the second step to replace the parameters passed in args*/
- /* 进行第二步，对args传进的参数进行替换*/
- task match {
- case requestPersistTask:RequestPersistTask => TaskUtils.getVariableMap(requestPersistTask.getParams.map{case (k, v) => k -> v.asInstanceOf[Any]}) match {
- case args:java.util.Map[String, Any] =>
- //Implicit conversion to scala map for subsequent operations(隐式转成scala的Map，方便后续操作)
- val scalaArgs:mutable.Map[String, Any] = args
- scalaArgs foreach {
- case (key, value) => key match {
- case RUN_DATE => if (nameAndType.get(RUN_DATE).isEmpty) {
- val run_date_str = value.asInstanceOf[String]
- if (StringUtils.isNotEmpty(run_date_str)){
- run_date = new CustomDateType(run_date_str, false)
- nameAndType(RUN_DATE) = DateType(run_date)
- }
- }
- case _ => if (nameAndType.get(key).isEmpty){
- val value_str = value.asInstanceOf[String]
- if(StringUtils.isNotEmpty(value_str)){
- /*if ((allCatch opt value_str.toLong).isDefined) {
- nameAndType(key) = LongType(value_str.toLong)
- } else*/ if ((allCatch opt value_str.toDouble).isDefined) {
- nameAndType(key) = DoubleValue(value_str.toDouble)
- } else {
- nameAndType(key) = StringType(value_str)
- }
- }
- }
+ case _ => if (nameAndType.get(key).isEmpty && StringUtils.isNotEmpty(value)) {
+ if ((allCatch opt value.toDouble).isDefined) {
+ nameAndType(key) = DoubleValue(value.toDouble)
+ } else {
+ nameAndType(key) = StringType(value)
}
}
- case _ =>
}
+ }
+
+ //The first step is to replace the variable from code
+ //第一步来自code的变量替换
+ putNameAndType(nameAndValue)
+
+ task match {
+ case requestPersistTask:RequestPersistTask =>
+ /* Perform the second step to replace the parameters passed in args*/
+ /* 进行第二步，对args传进的参数进行替换*/
+ val variableMap = TaskUtils.getVariableMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]])
+ .map{case (k, v) => k -> v.asInstanceOf[String]}
+ putNameAndType(variableMap)
case _ =>
}
- /* Go to the third step and take the user's parameters to the cloud-publicservice module.*/
- /*进行第三步，向cloud-publicservice模块去拿用户的参数*/
+ /* Go to the four step and take the user's parameters to the cloud-publicservice module.*/
+ /*进行第四步，向cloud-publicservice模块去拿用户的参数*/
val sender = Sender.getSender(EntranceConfiguration.CLOUD_CONSOLE_VARIABLE_SPRING_APPLICATION_NAME.getValue)
task match {
case requestPersistTask:RequestPersistTask =>
val umUser:String = requestPersistTask.getUmUser
val creator:String = requestPersistTask.getRequestApplicationName
val runType:String = requestPersistTask.getRunType
-// val requestQueryAppConfig:RequestQueryAppConfig = RequestQueryAppConfig(umUser, creator, runType)
-// val responseQueryConfig = sender.ask(requestQueryAppConfig).asInstanceOf[ResponseQueryConfig]
-// val keyAndValue = responseQueryConfig.getKeyAndValue
val requestQueryAppVariable:RequestQueryAppVariable = RequestQueryAppVariable(umUser, creator, runType)
val response:ResponseQueryVariable = sender.ask(requestQueryAppVariable).asInstanceOf[ResponseQueryVariable]
val keyAndValue = response.getKeyAndValue
val keyAndValueScala:mutable.Map[String, String] = keyAndValue
- keyAndValueScala foreach {
- case (key, value) => key match {
- case RUN_DATE => if (nameAndType.get(RUN_DATE).isEmpty) {
- val run_date_str = value.asInstanceOf[String]
- if (StringUtils.isNotEmpty(run_date_str)){
- run_date = new CustomDateType(run_date_str, false)
- nameAndType(RUN_DATE) = DateType(run_date)
- }
- }
- case _ => if (nameAndType.get(key).isEmpty){
- val value_str = value.asInstanceOf[String]
- if(StringUtils.isNotEmpty(value_str)){
- /*if ((allCatch opt value_str.toLong).isDefined) {
- nameAndType(key) = LongType(value_str.toLong)
- } else*/ if ((allCatch opt value_str.toDouble).isDefined) {
- nameAndType(key) = DoubleValue(value_str.toDouble)
- } else {
- nameAndType(key) = StringType(value_str)
- }
- }
- }
- }
- }
+ putNameAndType(keyAndValueScala)
case _ =>
}
/*The last step, if you have not set run_date, then it is the default */
/*最后一步，如果都没有设置run_date，那么就是默认*/
- if (nameAndType.get(RUN_DATE).isEmpty){
+ if (nameAndType.get(RUN_DATE).isEmpty || null == run_date){
run_date = new CustomDateType(getYesterday(false), false)
nameAndType(RUN_DATE) = DateType(new CustomDateType(run_date.toString, false))
}
@@ -213,8 +161,8 @@
*/
def parserVar(code: String, nameAndType: mutable.Map[String, VariableType]): String = {
- val codeReg = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r
- val calReg = "(\\s*[A-Za-z][A-Za-z0-9_]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r
+ val codeReg = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r
+ val calReg = "(\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r
val parseCode = new StringBuilder
val codes = codeReg.split(code)
val expressionCache = mutable.HashSet[String]()
@@ -369,11 +317,11 @@
val nameSet = res(0).split("@set")
if (nameSet != null && nameSet.length == 2) {
val name = nameSet(1).trim
- if (nameAndValue.getOrElse(name, null) == null) {
- nameAndValue(name) = res(1).trim
- } else {
- throw VarSubstitutionException(20043, s"$name is defined repeatedly")
- }
+ //if (nameAndValue.getOrElse(name, null) == null) {
+ nameAndValue(name) = res(1).trim
+ // } else {
+ // throw VarSubstitutionException(20043, s"$name is defined repeatedly")
+ // }
}
} else {
if (res.length > 2) {
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
index 00ac48a..e6c54a9 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala
@@ -21,7 +21,7 @@
import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.common.log.LogUtils
-import com.webank.wedatasphere.linkis.common.utils.Logging
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.{PythonCodeCheckException, ScalaCodeCheckException}
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import org.apache.commons.lang.StringUtils
@@ -82,6 +82,8 @@
val SQL_APPEND_LIMIT:String = " limit " + SQL_DEFAULT_LIMIT.getValue
val DROP_TABLE_SQL = "\\s*drop\\s+table\\s+\\w+\\s*"
val CREATE_DATABASE_SQL = "\\s*create\\s+database\\s+\\w+\\s*"
+ private val LINE_BREAK = "\n"
+ private val COMMENT_FLAG = "--"
val SET_OWN_USER = "set owner user"
private val LIMIT:String = "limit"
private val LIMIT_UPPERCASE:String = "LIMIT"
@@ -131,7 +133,9 @@
if (StringUtils.isEmpty(code)) {
return false
}
- code.trim.split("\\s+")(0).equalsIgnoreCase("select")
+ //如果一段sql是 --xxx回车select * from default.users，那么他也是select语句
+ val realCode = cleanComment(code)
+ realCode.trim.split("\\s+")(0).equalsIgnoreCase("select")
}
def continueWhenError = false
@@ -141,7 +145,24 @@
return false
}
val realCode = cmd.trim
- if (realCode.toLowerCase().contains(LIMIT)) false else true
+ //limit往往就是在sql语句中最后的，所以需要进行最后的判断
+ val arr = realCode.split("\\s+")
+ val words = new ArrayBuffer[String]()
+ arr foreach {
+ w => w.split("\n") foreach (words += _)
+ }
+ val a = words.toArray
+ val length = a.length
+ val second_last = a(length - 2)
+ !"limit".equals(second_last.toLowerCase())
+ }
+
+ private def cleanComment(sql:String):String = {
+ val cleanSql = new StringBuilder
+ sql.trim.split(LINE_BREAK) foreach {
+ singleSql => if (!singleSql.trim().startsWith(COMMENT_FLAG)) cleanSql.append(singleSql).append(LINE_BREAK)
+ }
+ cleanSql.toString().trim
}
def isSelectOverLimit(cmd: String): Boolean = {
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
index 5a30c86..5c4c95b 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala
@@ -31,7 +31,7 @@
override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match{
case requestPersistTask:RequestPersistTask =>
val error = new StringBuilder
- requestPersistTask.getEngineType match {
+ requestPersistTask.getRunType match {
case "python" | "pyspark" =>
Utils.tryThrow(PythonExplain.authPass(requestPersistTask.getExecutionCode, error)){
case PythonCodeCheckException(errCode,errDesc) =>
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
index 3b6076d..a0534d1 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala
@@ -17,7 +17,6 @@
package com.webank.wedatasphere.linkis.entrance.interceptor.impl
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
-import com.webank.wedatasphere.linkis.entrance.interceptor.exception.LimitCheckException
import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask
import com.webank.wedatasphere.linkis.protocol.task.Task
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
index dbf71be..557504f 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala
@@ -18,7 +18,6 @@
import java.lang
-import com.webank.wedatasphere.linkis.common.exception.ErrorException
import com.webank.wedatasphere.linkis.common.utils.Utils
import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor
import com.webank.wedatasphere.linkis.entrance.interceptor.exception.ScalaCodeCheckException
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
index 33b13ed..3a6beb5 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala
@@ -33,8 +33,8 @@
override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = {
task match {
case requestPersistTask: RequestPersistTask =>
- requestPersistTask.getExecuteApplicationName.toLowerCase() match {
- case "spark" | "scala" => val stringBuilder:StringBuilder = new StringBuilder()
+ requestPersistTask.getRunType.toLowerCase() match {
+ case "scala" => val stringBuilder:StringBuilder = new StringBuilder()
val isAuth = SparkExplain.authPass(requestPersistTask.getExecutionCode, stringBuilder)
if (!isAuth){
throw CodeCheckException(20050, "spark code check failed")
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala
index 9336729..3c15e1e 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala
@@ -33,11 +33,12 @@
def getCache:Option[Cache] = Some(sharedCache)
private def cache(msg:String): Unit = {
- val removed = sharedCache.cachedLogs.add(msg)
- if (removed != null){
- this synchronized{
+ this synchronized {
+ val removed = sharedCache.cachedLogs.add(msg)
+ if (removed != null){
val logs = sharedCache.cachedLogs.toList
val sb = new StringBuilder
+ sb.append(removed).append("\n")
logs.filter(_ != null).foreach(log => sb.append(log).append("\n"))
sharedCache.cachedLogs.fakeClear()
super.write(sb.toString())
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala
index c9142f1..31f1a12 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala
@@ -16,18 +16,6 @@
package com.webank.wedatasphere.linkis.entrance.log
-import java.io.{BufferedReader, FileInputStream, InputStreamReader}
-import java.util
-import java.util.concurrent.TimeUnit
-
-import com.webank.wedatasphere.linkis.common.io.FsPath
-import com.webank.wedatasphere.linkis.common.utils.Utils
-import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
-import com.webank.wedatasphere.linkis.storage.FSFactory
-import org.slf4j.{Logger, LoggerFactory}
-
-import scala.collection.mutable.ArrayBuffer
-
/**
* Created by enjoyyin on 2018/9/4.
*/
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala
index 52a8a45..c1be9a5 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala
@@ -56,14 +56,13 @@
}
val formatCode = params.get(TaskConstant.FORMATCODE).asInstanceOf[Boolean]
var creator = params.get(TaskConstant.REQUESTAPPLICATIONNAME).asInstanceOf[String]
- val scriptPath = params.get(TaskConstant.SCRIPTPATH).asInstanceOf[String]
val source = params.getOrDefault(TaskConstant.SOURCE,new util.HashMap[String,String]()).asInstanceOf[util.Map[String,String]]
val executeApplicationName = params.get(TaskConstant.EXECUTEAPPLICATIONNAME).asInstanceOf[String]
if (StringUtils.isEmpty(creator)) creator = EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue
if (StringUtils.isEmpty(executeApplicationName)) throw new EntranceIllegalParamException(20006, "param executeApplicationName can not be empty or null")
/* When the execution type is IDE, executionCode and scriptPath cannot be empty at the same time*/
/*当执行类型为IDE的时候，executionCode和scriptPath不能同时为空*/
- if (EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue.equals(creator) && StringUtils.isEmpty(scriptPath) &&
+ if (EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue.equals(creator) && StringUtils.isEmpty(source.get(TaskConstant.SCRIPTPATH)) &&
StringUtils.isEmpty(executionCode))
throw new EntranceIllegalParamException(20007, "param executionCode and scriptPath can not be empty at the same time")
var runType:String = null
@@ -74,20 +73,7 @@
if (formatCode) executionCode = format(executionCode)
task.setExecutionCode(executionCode)
}
- if (source.isEmpty) source.put(TaskConstant.SCRIPTPATH,scriptPath)
task.setSource(source)
- if (StringUtils.isNotEmpty(scriptPath)) {
- task.setScriptPath(scriptPath)
-// val strings = StringUtils.split(scriptPath, ".")
-// if (strings.length >= 2) {
-// /*获得执行脚本文件的后缀名,如果不能从一个执行脚本的后缀名判断出类型，可能需要报错*/
-// //todo If you can't get it from the file suffix name(如果不能从文件后缀名得到)
-// runType = strings(strings.length - 1)
-// if (ParserUtils.getCorrespondingType(runType) != null) runType = ParserUtils.getCorrespondingType(runType)
-// else logger.warn("未能找到相应的执行类型:" + runType)
-// }
-// else logger.warn("scriptPath:" + scriptPath + "没有后缀名, 无法判断任务是哪种类型")
- }
task.setEngineType(runType)
//为了兼容代码，让engineType和runType都有同一个属性
task.setRunType(runType)
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala
index 32a0990..13bd941 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala
@@ -21,8 +21,6 @@
import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob
import com.webank.wedatasphere.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse}
import com.webank.wedatasphere.linkis.scheduler.queue.Job
-import com.webank.wedatasphere.linkis.storage.FSFactory
-import com.webank.wedatasphere.linkis.storage.fs.FileSystem
import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter}
import com.webank.wedatasphere.linkis.storage.utils.{FileSystemUtils, StorageUtils}
import org.apache.commons.io.IOUtils
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala
index 29b760b..a436caa 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala
@@ -19,7 +19,6 @@
import java.io.{Closeable, Flushable}
import com.webank.wedatasphere.linkis.common.exception.ErrorException
-import com.webank.wedatasphere.linkis.entrance.exception.EntranceIllegalParamException
import com.webank.wedatasphere.linkis.protocol.task.Task
/**
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
index 47e6fd5..620af1c 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala
@@ -19,6 +19,7 @@
import java.util
import javax.servlet.http.HttpServletRequest
+import javax.ws.rs.QueryParam
import javax.ws.rs.core.{Context, Response}
import org.springframework.web.bind.annotation.{PathVariable, RequestBody, RequestMapping, RequestMethod}
@@ -34,7 +35,7 @@
// def get(@PathVariable("id") id: String): Response
@RequestMapping(value = Array("/entrance/{id}/status"), method = Array(RequestMethod.GET))
- def status(@PathVariable("id") id: String): Response
+ def status(@PathVariable("id") id: String, @QueryParam("taskID") taskID:String): Response
@RequestMapping(value = Array("/entrance/{id}/progress"), method = Array(RequestMethod.POST))
@@ -48,7 +49,7 @@
def log(@Context req: HttpServletRequest, @PathVariable("id") id: String): Response
@RequestMapping(value = Array("/entrance/{id}/kill"), method = Array(RequestMethod.POST))
- def kill(@PathVariable("id") id: String): Response
+ def kill(@PathVariable("id") id: String, @QueryParam("taskID") taskID:Long): Response
@RequestMapping(value = Array("/entrance/{id}/pause"), method = Array(RequestMethod.POST))
def pause(@PathVariable("id") id: String): Response
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala
index d7fdf6b..7913bc3 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala
@@ -16,11 +16,10 @@
package com.webank.wedatasphere.linkis.entrance.scheduler
-import com.webank.wedatasphere.linkis.common.conf.CommonVars
import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob
import com.webank.wedatasphere.linkis.entrance.persistence.HaPersistenceTask
-import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfig, RequestQueryGlobalConfig, ResponseQueryConfig}
+import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfig, ResponseQueryConfig}
import com.webank.wedatasphere.linkis.rpc.Sender
import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelGroup
import com.webank.wedatasphere.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent}
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala
index 0858d85..2d02502 100644
--- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala
@@ -21,8 +21,6 @@
import com.webank.wedatasphere.linkis.scheduler.event.{ScheduleEvent, SchedulerEventListener}
import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorManager
import com.webank.wedatasphere.linkis.scheduler.queue.{ConsumerManager, GroupFactory}
-import org.springframework.beans.factory.annotation.Autowired
-import org.springframework.stereotype.Component
/**
* Created by enjoyyin on 2019/1/22.
diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala
new file mode 100644
index 0000000..f330fcd
--- /dev/null
+++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala
@@ -0,0 +1,87 @@
+package com.webank.wedatasphere.linkis.entrance.utils
+
+import java.util
+
+import com.google.gson.Gson
+import com.webank.wedatasphere.linkis.common.exception.ErrorException
+import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils}
+import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration
+import com.webank.wedatasphere.linkis.entrance.exception.JobHistoryFailedException
+import com.webank.wedatasphere.linkis.protocol.query.cache.{CacheTaskResult, RequestReadCache}
+import com.webank.wedatasphere.linkis.protocol.query.{RequestPersistTask, RequestQueryTask, RequestUpdateTask, ResponsePersist}
+import com.webank.wedatasphere.linkis.rpc.Sender
+import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState
+
+/**
+ * created by cooperyang on 2020/1/2
+ * Description:
+ */
+object JobHistoryHelper extends Logging{
+
+ private val sender = Sender.getSender(EntranceConfiguration.QUERY_PERSISTENCE_SPRING_APPLICATION_NAME.getValue)
+
+ private val SUCCESS_FLAG = 0
+ private val TASK_MAP_KEY = "task"
+
+ def getCache(executionCode: String, engineType: String, user: String, readCacheBefore: Long): CacheTaskResult ={
+ val requestReadCache = new RequestReadCache(executionCode, engineType, user, readCacheBefore)
+ sender.ask(requestReadCache) match {
+ case c: CacheTaskResult => c
+ case _ => null
+ }
+ }
+
+ def getStatusByTaskID(taskID:Long):String = {
+ val task = getTaskByTaskID(taskID)
+ if (task == null) SchedulerEventState.Cancelled.toString else task.getStatus
+ }
+
+ /**
+ * 对于一个在内存中找不到这个任务的话，可以直接干掉
+ * @param taskID
+ */
+ def forceKill(taskID:Long):Unit = {
+ val requestUpdateTask = new RequestUpdateTask
+ requestUpdateTask.setTaskID(taskID)
+ requestUpdateTask.setStatus("Cancelled")
+ sender.ask(requestUpdateTask)
+ }
+
+ private def getTaskByTaskID(taskID:Long):RequestPersistTask = {
+ val requestQueryTask = new RequestQueryTask()
+ requestQueryTask.setTaskID(taskID)
+ requestQueryTask.setSource(null)
+ val task = Utils.tryCatch{
+ val taskResponse = sender.ask(requestQueryTask)
+ taskResponse match {
+ case responsePersist:ResponsePersist => val status = responsePersist.getStatus
+ if (status != SUCCESS_FLAG){
+ logger.error(s"query from jobHistory status failed, status is $status")
+ throw JobHistoryFailedException("query from jobHistory status failed")
+ }else{
+ val data = responsePersist.getData
+ data.get(TASK_MAP_KEY) match {
+ case tasks:util.List[util.Map[String, Object]] => tasks.get(0) match {
+ case map:util.Map[String, Object] => val gson = new Gson()
+ val json = gson.toJson(map)
+ val requestPersistTask = gson.fromJson(json, classOf[RequestPersistTask])
+ requestPersistTask
+ case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct RequestPersistTask type taskId is $taskID")
+ }
+ case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct List type taskId is $taskID")
+ }
+ }
+ case _ => logger.error("get query response incorrectly")
+ throw JobHistoryFailedException("get query response incorrectly")
+ }
+ }{
+ case errorException:ErrorException => throw errorException
+ case e:Exception => val e1 = JobHistoryFailedException(s"query taskId $taskID error")
+ e1.initCause(e)
+ throw e
+ }
+ task
+ }
+
+
+}
diff --git a/ujes/entranceclient/pom.xml b/ujes/entranceclient/pom.xml
index 01158a6..8111b2e 100644
--- a/ujes/entranceclient/pom.xml
+++ b/ujes/entranceclient/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-entranceclient</artifactId>
@@ -40,6 +40,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.webank.wedatasphere.linkis</groupId>
+ <artifactId>linkis-cloudRPC</artifactId>
+ <version>${linkis.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/ujes/jdbc/pom.xml b/ujes/jdbc/pom.xml
index 1523904..80d7790 100644
--- a/ujes/jdbc/pom.xml
+++ b/ujes/jdbc/pom.xml
@@ -1,5 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2019 WeBank
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -7,7 +20,7 @@
<parent>
<artifactId>linkis</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<artifactId>linkis-ujes-jdbc</artifactId>
@@ -15,7 +28,7 @@
<dependency>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis-ujes-client</artifactId>
- <version>0.9.3</version>
+ <version>${linkis.version}</version>
<!--<exclusions>-->
<!--<exclusion>-->
<!--<groupId>com.google.guava</groupId>-->
diff --git a/userControl/pom.xml b/userControl/pom.xml
index fca6746..67c41c1 100644
--- a/userControl/pom.xml
+++ b/userControl/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>com.webank.wedatasphere.linkis</groupId>
<artifactId>linkis</artifactId>
- <version>0.9.3</version>
+ <version>0.9.4</version>
</parent>
<modelVersion>4.0.0</modelVersion>
