task log print optimize
diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java
index c1479ef..f3c3e9e 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java
@@ -110,15 +110,6 @@
JobRequest jobReq = ((EntranceJob) job).getJobRequest();
Long jobReqId = jobReq.getId();
ModuleUserUtils.getOperationUser(req, "execute task,id: " + jobReqId);
- pushLog(
- LogUtils.generateInfo(
- "You have submitted a new job, script code (after variable substitution) is"),
- job);
- pushLog(
- "************************************SCRIPT CODE************************************", job);
- pushLog(jobReq.getExecutionCode(), job);
- pushLog(
- "************************************SCRIPT CODE************************************", job);
String execID =
ZuulEntranceUtils.generateExecID(
job.getId(),
@@ -164,15 +155,6 @@
ModuleUserUtils.getOperationUser(req, "submit jobReqId: " + jobReqId);
pushLog(
LogUtils.generateInfo(
- "You have submitted a new job, script code (after variable substitution) is"),
- job);
- pushLog(
- "************************************SCRIPT CODE************************************", job);
- pushLog(jobRequest.getExecutionCode(), job);
- pushLog(
- "************************************SCRIPT CODE************************************", job);
- pushLog(
- LogUtils.generateInfo(
"Your job is accepted, jobID is "
+ job.getId()
+ " and jobReqId is "
@@ -594,9 +576,9 @@
logger.error("kill job {} failed ", job.get().getId(), t);
message =
Message.error(
- "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)");
+ "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)",
+ t);
message.setMethod("/api/entrance/" + id + "/kill");
- message.setStatus(1);
}
}
messages.add(message);
@@ -678,10 +660,11 @@
logger.error("kill job {} failed ", job.get().getId(), t);
message =
Message.error(
- "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"
- + "message: "
- + t.getMessage());
+ "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败) with error:"
+ + t.getMessage(),
+ t);
message.setMethod("/api/entrance/" + id + "/kill");
+ message.setStatus(1);
}
}
return message;
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala
index 627ab82..34bd6ea 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala
@@ -18,7 +18,6 @@
package org.apache.linkis.entrance.interceptor.impl
import org.apache.linkis.common.utils.CodeAndRunTypeUtils
-import org.apache.linkis.entrance.conf.EntranceConfiguration
import org.apache.linkis.entrance.interceptor.EntranceInterceptor
import org.apache.linkis.governance.common.entity.job.JobRequest
import org.apache.linkis.manager.label.utils.LabelUtil
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala
index 8436ccc..35b40db 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala
@@ -57,6 +57,8 @@
private val sy = Pattern.compile("sys\\.")
private val scCancelAllJobs = Pattern.compile("sc\\.cancelAllJobs(\\s*)")
private val runtime = Pattern.compile("Runtime\\.getRuntime")
+ private val LINE_BREAK = "\n"
+ private val LOG: Logger = LoggerFactory.getLogger(getClass)
override def authPass(code: String, error: StringBuilder): Boolean = {
if (EntranceConfiguration.SKIP_AUTH.getHotValue()) {
@@ -99,6 +101,7 @@
private val LIMIT: String = "limit"
private val LIMIT_UPPERCASE: String = "LIMIT"
private val IDE_ALLOW_NO_LIMIT = "--set wds.linkis.engine.no.limit.allow=true"
+ private val LOG: Logger = LoggerFactory.getLogger(getClass)
override def authPass(code: String, error: StringBuilder): Boolean = {
true
@@ -131,6 +134,8 @@
.generateWarn("please pay attention ,SQL full export mode opened(请注意,SQL全量导出模式打开)\n")
)
}
+ var isFirstTimePrintingLimit = true
+ var isFirstTimePrintingOverLimit = true
if (tempCode.contains("""\;""")) {
val semicolonIndexes = findRealSemicolonIndex(tempCode)
var oldIndex = 0
@@ -140,20 +145,27 @@
if (isSelectCmd(singleCode)) {
val trimCode = singleCode.trim
if (isSelectCmdNoLimit(trimCode) && !isNoLimitAllowed) {
- logAppender.append(
- LogUtils.generateWarn(
- s"You submitted a sql without limit, DSS will add limit 5000 to your sql"
- ) + "\n"
- )
+ if (isFirstTimePrintingLimit) {
+ logAppender.append(
+ LogUtils.generateWarn(
+ s"You submitted a sql without limit, DSS will add limit 5000 to your sql"
+ ) + "\n"
+ )
+ isFirstTimePrintingLimit = false
+ }
+ // 将注释先干掉,然后再进行添加limit
val realCode = cleanComment(trimCode)
fixedCode += (realCode + SQL_APPEND_LIMIT)
} else if (isSelectOverLimit(singleCode) && !isNoLimitAllowed) {
val trimCode = singleCode.trim
- logAppender.append(
- LogUtils.generateWarn(
- s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000"
- ) + "\n"
- )
+ if (isFirstTimePrintingOverLimit) {
+ logAppender.append(
+ LogUtils.generateWarn(
+ s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000"
+ ) + "\n"
+ )
+ isFirstTimePrintingOverLimit = false
+ }
fixedCode += repairSelectOverLimit(trimCode)
} else {
fixedCode += singleCode.trim
@@ -167,20 +179,27 @@
if (isSelectCmd(singleCode)) {
val trimCode = singleCode.trim
if (isSelectCmdNoLimit(trimCode) && !isNoLimitAllowed) {
- logAppender.append(
- LogUtils.generateWarn(
- s"You submitted a sql without limit, DSS will add limit 5000 to your sql"
- ) + "\n"
- )
+ if (isFirstTimePrintingLimit) {
+ logAppender.append(
+ LogUtils.generateWarn(
+ s"You submitted a sql without limit, DSS will add limit 5000 to your sql"
+ ) + "\n"
+ )
+ isFirstTimePrintingLimit = false
+ }
+ // 将注释先干掉,然后再进行添加limit
val realCode = cleanComment(trimCode)
fixedCode += (realCode + SQL_APPEND_LIMIT)
} else if (isSelectOverLimit(singleCode) && !isNoLimitAllowed) {
val trimCode = singleCode.trim
- logAppender.append(
- LogUtils.generateWarn(
- s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000"
- ) + "\n"
- )
+ if (isFirstTimePrintingOverLimit) {
+ logAppender.append(
+ LogUtils.generateWarn(
+ s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000"
+ ) + "\n"
+ )
+ isFirstTimePrintingOverLimit = false
+ }
fixedCode += repairSelectOverLimit(trimCode)
} else {
fixedCode += singleCode.trim
@@ -210,6 +229,8 @@
array.toArray
}
+ private def addNoLimit(code: String) = code + NO_LIMIT_STRING
+
protected def needNoLimit(code: String): Boolean = code.endsWith(NO_LIMIT_STRING)
def isSelectCmd(code: String): Boolean = {
@@ -217,16 +238,17 @@
return false
}
val realCode = cleanComment(code)
- realCode.trim.split("\\s+")(0).toLowerCase().contains("select")
+ realCode.trim.split("\\s+")(0).toLowerCase(Locale.getDefault).contains("select")
}
- def continueWhenError: Boolean = false
+ // def continueWhenError = false
def isSelectCmdNoLimit(cmd: String): Boolean = {
if (StringUtils.isEmpty(cmd)) {
return false
}
val realCode = cmd.trim
+ // limit is often the last in a sql statement, so you need to make a final judgment
val arr = realCode.split("\\s+")
val words = new ArrayBuffer[String]()
arr foreach { w =>
@@ -235,8 +257,10 @@
val a = words.toArray
val length = a.length
if (a.length > 1) {
- val second_last = a(length - 2)
- !"limit".equals(second_last.toLowerCase())
+ val second_last = a(length - 2).toLowerCase(Locale.getDefault)
+ // for some case eg:"SELECT * from dual WHERE (1=1)LIMIT 1;"
+ val result = !("limit".equals(second_last) || second_last.contains(")limit"))
+ result
} else {
false
}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
index 0487a23..72d4030 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala
@@ -41,10 +41,26 @@
LogUtils.generateInfo("Program is substituting variables for you") + "\n"
)
val codeType = LabelUtil.getCodeType(jobRequest.getLabels)
- jobRequest.setExecutionCode(CustomVariableUtils.replaceCustomVar(jobRequest, codeType))
+ val realCode = CustomVariableUtils.replaceCustomVar(jobRequest, codeType)
+ jobRequest.setExecutionCode(realCode)
logAppender.append(
LogUtils.generateInfo("Variables substitution ended successfully") + "\n"
)
+ // print code after variables substitution
+ logAppender.append(
+ LogUtils.generateInfo(
+ "You have submitted a new job, script code (after variable substitution) is"
+ ) + "\n"
+ );
+ logAppender.append(
+ "************************************SCRIPT CODE************************************" + "\n"
+ )
+ logAppender.append(realCode);
+ logAppender.append("\n");
+ logAppender.append(
+ "************************************SCRIPT CODE************************************" + "\n"
+ );
+
jobRequest
} {
case e: VarSubstitutionException =>
diff --git a/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/SQLExplainTest.java b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/SQLExplainTest.java
new file mode 100644
index 0000000..c5efb56
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/SQLExplainTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.entrance.interceptor.impl;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class SQLExplainTest {
+
+ @Test
+ void isSelectCmdNoLimit() {
+
+ String code = "SELECT * from dual WHERE (1=1)LIMIT 1;";
+ boolean res = SQLExplain.isSelectCmdNoLimit(code);
+ Assertions.assertEquals(false, res);
+
+ code = "SELECT * from dual";
+ res = SQLExplain.isSelectCmdNoLimit(code);
+ Assertions.assertEquals(true, res);
+
+ code = "SELECT * from dual LIMIT 1;";
+ res = SQLExplain.isSelectCmdNoLimit(code);
+ Assertions.assertEquals(false, res);
+ }
+
+ @Test
+ void isSelectOverLimit() {
+ String code = "SELECT * from dual WHERE (1=1)LIMIT 5001;";
+ boolean res = SQLExplain.isSelectOverLimit(code);
+ Assertions.assertEquals(true, res);
+
+ code = "SELECT * from dual";
+ res = SQLExplain.isSelectOverLimit(code);
+ Assertions.assertEquals(false, res);
+
+ code = "SELECT * from dual LIMIT 4000;";
+ res = SQLExplain.isSelectOverLimit(code);
+ Assertions.assertEquals(false, res);
+ }
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java
new file mode 100644
index 0000000..34826ff
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.linkis.entrance.interceptor.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.linkis.cs.common.entity.source.ContextID;
+import org.apache.linkis.entrance.log.Cache;
+import org.apache.linkis.entrance.log.HDFSCacheLogWriter;
+import org.apache.linkis.rpc.BaseRPCSender;
+import org.apache.linkis.rpc.Sender;
+import org.apache.linkis.storage.fs.FileSystem;
+import org.apache.linkis.storage.fs.impl.LocalFileSystem;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import oshi.util.FileUtil;
+
+class TestHDFSCacheLogWriter {
+
+
+ @Test
+ void write() throws IOException {
+
+ Cache cache = new Cache(5);
+ String fileName= UUID.randomUUID().toString().replace("-", "")+"-test.log";
+ String logPath = System.getProperty("java.io.tmpdir")+ File.separator+fileName;
+ System.out.println(logPath);
+ String chartSet="utf-8";
+ String username=System.getProperty("user.name");
+
+ File file=new File(logPath);
+ file.createNewFile();
+
+ HDFSCacheLogWriter logWriter =new HDFSCacheLogWriter(
+ //"D:\\DataSphere\\linkis\\docs\\test.log",
+ logPath,
+ chartSet,
+ cache,
+ username
+ );
+
+ String[] msgArr =new String[]{"1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18"};
+
+ List<String> msgList = new ArrayList<String>(Arrays.asList(msgArr));
+ String msg=String.join("\n", msgList);
+
+ logWriter.write(msg);
+ logWriter.flush();
+
+ List<String> list = FileUtil.readFile(logPath);
+ String res=String.join("\n", list);
+
+ res=res.replace("\n\n","\n");
+ res=StringUtils.strip(res, " \n");
+ Assertions.assertEquals(res,msg);
+
+
+ }
+}
\ No newline at end of file
