[Fix-17520][TaskExecutor] Fix not clear task exec path when set development.state=false in common.properties (#17523)
diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
index 81143ae..9e5adea 100644
--- a/dolphinscheduler-common/src/main/resources/common.properties
+++ b/dolphinscheduler-common/src/main/resources/common.properties
@@ -81,7 +81,7 @@
# network IP gets priority, default: inner outer
#dolphin.scheduler.network.priority.strategy=default
-# development state
+# development state. it may delete task exec path when task finished, or keep it for debug reason in development
development.state=false
# If the shell process is still active after this timeout value (in seconds), then will use kill -9 to kill it
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutor.java
index 4dafabf..738b36f 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutor.java
@@ -78,6 +78,11 @@
}
@Override
+ public void finalizeTask() {
+ // do nothing for now
+ }
+
+ @Override
public String toString() {
return "LogicTaskExecutor{" +
"id=" + taskExecutionContext.getTaskInstanceId() +
diff --git a/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/ITaskExecutor.java b/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/ITaskExecutor.java
index f62d9ab..2dd25b8 100644
--- a/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/ITaskExecutor.java
+++ b/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/ITaskExecutor.java
@@ -70,6 +70,11 @@
void kill();
/**
+ * finalize the task executor.
+ */
+ void finalizeTask();
+
+ /**
* Get the EventBus belongs to the task executor.
*/
TaskExecutorEventBus getTaskExecutorEventBus();
diff --git a/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/container/AbstractTaskExecutorContainer.java b/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/container/AbstractTaskExecutorContainer.java
index fd87643..4eac0e2 100644
--- a/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/container/AbstractTaskExecutorContainer.java
+++ b/dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/container/AbstractTaskExecutorContainer.java
@@ -98,6 +98,8 @@
}
log.info(FINALIZE_SESSION_MARKER, FINALIZE_SESSION_MARKER.toString());
pushTaskExecutorLogToRemote(taskExecutor);
+ // [Fix-17520]
+ taskExecutor.finalizeTask();
}
@Override
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java
index d6139b2..1d4c7b0 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/executor/PhysicalTaskExecutor.java
@@ -17,7 +17,9 @@
package org.apache.dolphinscheduler.server.worker.executor;
+import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
@@ -105,6 +107,18 @@
}
@Override
+ public void finalizeTask() {
+ clearTaskInstanceWorkingDirectoryIfNeeded();
+ }
+
+ private void clearTaskInstanceWorkingDirectoryIfNeeded() {
+ boolean isDevelopment = PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
+ if (!isDevelopment) {
+ TaskExecutionContextUtils.clearTaskInstanceWorkingDirectory(taskExecutionContext);
+ }
+ }
+
+ @Override
protected void initializeTaskContext() {
super.initializeTaskContext();
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
index 79536bb..bdcbdbe 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
@@ -29,6 +29,7 @@
import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics;
import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.nio.file.Files;
@@ -108,4 +109,16 @@
return resourceContext;
}
+ public static void clearTaskInstanceWorkingDirectory(TaskExecutionContext taskExecutionContext) {
+ final String execPath = taskExecutionContext.getExecutePath();
+ try {
+ if (StringUtils.isNotEmpty(execPath)) {
+ FileUtils.deleteFile(execPath);
+ log.info("Deleted task exec directory: {}", execPath);
+ }
+ } catch (Exception e) {
+ log.warn("Failed to delete task exec directory.", e);
+ }
+ }
+
}
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
index 3fa08e3..36db668 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
@@ -55,4 +55,23 @@
FileUtils.deleteFile(taskWorkingDirectory);
}
}
+
+ @Test
+ void clearTaskInstanceWorkingDirectory() throws IOException {
+ TaskExecutionContext taskExecutionContext = new TaskExecutionContext();
+ taskExecutionContext.setTaskInstanceId(1);
+
+ TaskExecutionContextUtils.createTaskInstanceWorkingDirectory(taskExecutionContext);
+ String taskWorkingDirectory =
+ FileUtils.getTaskInstanceWorkingDirectory(taskExecutionContext.getTaskInstanceId());
+ Files.createFile(Paths.get(taskWorkingDirectory, "1.sh"));
+
+ // Test delete the working directory
+ TaskExecutionContextUtils.clearTaskInstanceWorkingDirectory(taskExecutionContext);
+ Assertions.assertFalse(Files.exists(Paths.get(taskWorkingDirectory)));
+
+ // Test do nothing if working directory is empty
+ TaskExecutionContext emptyExecPathContext = new TaskExecutionContext();
+ TaskExecutionContextUtils.clearTaskInstanceWorkingDirectory(emptyExecPathContext);
+ }
}
