Fix insertion cross space compaction recover null pointer exception (#12439)
* fix insertion cross space compaction null pointer
* add ut
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index 6f262a7..d813bd0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -85,6 +85,8 @@
super(databaseName, dataRegionId, 0L, tsFileManager, 0L, CompactionTaskPriorityType.NORMAL);
this.logFile = logFile;
this.needRecoverTaskInfoFromLogFile = true;
+ this.selectedSeqFiles = Collections.emptyList();
+ this.selectedUnseqFiles = new ArrayList<>(1);
}
private TsFileResource unseqFileToInsert;
@@ -242,6 +244,7 @@
File sourceTsFile = sourceFileIdentifiers.get(0).getFileFromDataDirsIfAnyAdjuvantFileExists();
if (sourceTsFile != null) {
unseqFileToInsert = new TsFileResource(sourceTsFile);
+ selectedUnseqFiles.add(unseqFileToInsert);
}
File targetTsFile = targetFileIdentifiers.get(0).getFileFromDataDirsIfAnyAdjuvantFileExists();
if (targetTsFile != null) {
@@ -288,7 +291,9 @@
return targetFile == null
|| !targetFile.tsFileExists()
|| !targetFile.resourceFileExists()
- || (unseqFileToInsert.modFileExists() && !targetFile.modFileExists());
+ || (unseqFileToInsert != null
+ && unseqFileToInsert.modFileExists()
+ && !targetFile.modFileExists());
}
private void rollback() throws IOException {
@@ -298,6 +303,9 @@
Collections.singletonList(targetFile), Collections.singletonList(unseqFileToInsert));
}
deleteCompactionModsFile(Collections.singletonList(unseqFileToInsert));
+ if (targetFile == null) {
+ return;
+ }
if (targetFile.tsFileExists()) {
FileMetrics.getInstance().deleteTsFile(true, Collections.singletonList(targetFile));
}
@@ -309,6 +317,9 @@
}
private void finishTask() throws IOException {
+ if (unseqFileToInsert == null) {
+ return;
+ }
if (recoverMemoryStatus && unseqFileToInsert.tsFileExists()) {
FileMetrics.getInstance().deleteTsFile(false, Collections.singletonList(unseqFileToInsert));
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
index e99d2c7..69b4bc9 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
@@ -345,6 +345,88 @@
Assert.assertTrue(targetFile.modFileExists());
}
+ @Test
+ public void testRecoverWithTargetFileNotExist()
+ throws IllegalPathException, IOException, MergeException {
+ IDeviceID d1 = new PlainDeviceID("root.testsg.d1");
+ IDeviceID d2 = new PlainDeviceID("root.testsg.d2");
+
+ TsFileResource seqResource1 = createTsFileResource("1-1-0-0.tsfile", true);
+ seqResource1.updateStartTime(d1, 10);
+ seqResource1.updateEndTime(d1, 20);
+ seqResource1.updateStartTime(d2, 20);
+ seqResource1.updateEndTime(d2, 30);
+ createTsFileByResource(seqResource1);
+ seqResource1.serialize();
+
+ TsFileResource seqResource2 = createTsFileResource("3-3-0-0.tsfile", true);
+ seqResource2.updateStartTime(d1, 30);
+ seqResource2.updateEndTime(d1, 40);
+ seqResource2.updateStartTime(d2, 40);
+ seqResource2.updateEndTime(d2, 50);
+ createTsFileByResource(seqResource2);
+ seqResource2.serialize();
+
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+
+ TsFileResource unseqResource1 = createTsFileResource("9-9-0-0.tsfile", false);
+ unseqResource1.updateStartTime(d1, 22);
+ unseqResource1.updateEndTime(d1, 25);
+ unseqResource1.updateStartTime(d2, 31);
+ unseqResource1.updateEndTime(d2, 37);
+ createTsFileByResource(unseqResource1);
+ unseqResource1.serialize();
+
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ deleteMap.put(((PlainDeviceID) d1).toStringID() + ".s1", new Pair<>(0L, 300L));
+ CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResource1, false);
+
+ unseqResources.add(unseqResource1);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("root.testsg", "0", 0, tsFileManager);
+ InsertionCrossCompactionTaskResource taskResource =
+ selector.selectOneInsertionTask(
+ new CrossSpaceCompactionCandidate(seqResources, unseqResources));
+ Assert.assertEquals(unseqResource1, taskResource.toInsertUnSeqFile);
+ Assert.assertEquals(seqResource1, taskResource.prevSeqFile);
+ Assert.assertEquals(seqResource2, taskResource.nextSeqFile);
+ Assert.assertEquals(unseqResource1, taskResource.firstUnSeqFileInParitition);
+
+ InsertionCrossSpaceCompactionTask task =
+ new InsertionCrossSpaceCompactionTask(new Phaser(), 0, tsFileManager, taskResource, 0);
+ TsFileResource targetFile = new TsFileResource(task.generateTargetFile());
+ File logFile =
+ new File(
+ targetFile.getTsFilePath() + CompactionLogger.INSERTION_COMPACTION_LOG_NAME_SUFFIX);
+
+ CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResource1, true);
+
+ try (SimpleCompactionLogger logger = new SimpleCompactionLogger(logFile)) {
+ logger.logSourceFile(taskResource.toInsertUnSeqFile);
+ logger.logTargetFile(targetFile);
+ logger.force();
+ }
+
+ // recover compaction, all target file and compaction.mods file should be deleted and source
+ // file should be existed
+ new InsertionCrossSpaceCompactionTask("root.testsg", "0", tsFileManager, logFile).recover();
+
+ Assert.assertTrue(unseqResource1.getTsFile().exists());
+ Assert.assertTrue(
+ new File(unseqResource1.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists());
+ Assert.assertTrue(unseqResource1.getModFile().exists());
+ Assert.assertFalse(unseqResource1.getCompactionModFile().exists());
+
+ Assert.assertFalse(targetFile.tsFileExists());
+ Assert.assertFalse(targetFile.resourceFileExists());
+ Assert.assertFalse(targetFile.modFileExists());
+ }
+
private TsFileResource createTsFileResource(String name, boolean seq) {
String filePath = (seq ? SEQ_DIRS : UNSEQ_DIRS) + File.separator + name;
TsFileResource resource = new TsFileResource();
