| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| package org.apache.iotdb.db.engine.compaction.cross; |
| |
| import org.apache.iotdb.commons.conf.IoTDBConstant; |
| import org.apache.iotdb.commons.exception.MetadataException; |
| import org.apache.iotdb.db.conf.IoTDBDescriptor; |
| import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest; |
| import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; |
| import org.apache.iotdb.db.engine.compaction.CompactionUtils; |
| import org.apache.iotdb.db.engine.compaction.log.CompactionLogger; |
| import org.apache.iotdb.db.engine.compaction.performer.ICompactionPerformer; |
| import org.apache.iotdb.db.engine.compaction.performer.impl.ReadPointCompactionPerformer; |
| import org.apache.iotdb.db.engine.compaction.task.CompactionTaskSummary; |
| import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer; |
| import org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils; |
| import org.apache.iotdb.db.engine.modification.ModificationFile; |
| import org.apache.iotdb.db.engine.storagegroup.TsFileManager; |
| import org.apache.iotdb.db.engine.storagegroup.TsFileName; |
| import org.apache.iotdb.db.engine.storagegroup.TsFileResource; |
| import org.apache.iotdb.db.exception.StorageEngineException; |
| import org.apache.iotdb.tsfile.common.constant.TsFileConstant; |
| import org.apache.iotdb.tsfile.exception.write.WriteProcessException; |
| import org.apache.iotdb.tsfile.utils.Pair; |
| |
| import org.junit.After; |
| import org.junit.Assert; |
| import org.junit.Before; |
| import org.junit.Test; |
| |
| import java.io.File; |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Map; |
| |
| import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_SOURCE_FILES; |
| import static org.apache.iotdb.db.engine.compaction.log.CompactionLogger.STR_TARGET_FILES; |
| import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; |
| |
| public class CrossSpaceCompactionExceptionTest extends AbstractCompactionTest { |
| |
| private final String oldThreadName = Thread.currentThread().getName(); |
| |
| @Before |
| public void setUp() throws IOException, WriteProcessException, MetadataException { |
| super.setUp(); |
| IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024); |
| Thread.currentThread().setName("pool-1-IoTDB-Compaction-1"); |
| } |
| |
| @After |
| public void tearDown() throws IOException, StorageEngineException { |
| super.tearDown(); |
| Thread.currentThread().setName(oldThreadName); |
| new CompactionConfigRestorer().restoreCompactionConfig(); |
| } |
| |
| @Test |
| public void testHandleWithAllSourceFilesExisted() throws Exception { |
| registerTimeseriesInMManger(4, 5, false); |
| createFiles(2, 2, 3, 300, 0, 0, 50, 50, false, true); |
| createFiles(2, 4, 5, 300, 700, 700, 50, 50, false, true); |
| createFiles(3, 3, 4, 200, 20, 10020, 30, 30, false, false); |
| createFiles(2, 1, 5, 100, 450, 20450, 0, 0, false, false); |
| TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath()); |
| tsFileManager.addAll(seqResources, true); |
| tsFileManager.addAll(unseqResources, false); |
| |
| List<TsFileResource> targetResources = |
| CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); |
| File compactionLogFile = |
| new File( |
| SEQ_DIRS, |
| targetResources.get(0).getTsFile().getName() |
| + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); |
| CompactionLogger compactionLogger = new CompactionLogger(compactionLogFile); |
| compactionLogger.logFiles(targetResources, STR_TARGET_FILES); |
| compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); |
| compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); |
| ICompactionPerformer performer = |
| new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources); |
| performer.setSummary(new CompactionTaskSummary()); |
| performer.perform(); |
| compactionLogger.close(); |
| CompactionExceptionHandler.handleException( |
| COMPACTION_TEST_SG, |
| compactionLogFile, |
| targetResources, |
| seqResources, |
| unseqResources, |
| tsFileManager, |
| 0, |
| false, |
| true); |
| // all source file should still exist |
| for (TsFileResource resource : seqResources) { |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| for (TsFileResource resource : unseqResources) { |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| // tmp target file, target file and target resource file should be deleted |
| for (TsFileResource resource : targetResources) { |
| Assert.assertFalse(resource.getTsFile().exists()); |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX, |
| TsFileConstant.TSFILE_SUFFIX)) |
| .exists()); |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX, |
| TsFileConstant.TSFILE_SUFFIX) |
| + TsFileResource.RESOURCE_SUFFIX) |
| .exists()); |
| } |
| Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); |
| Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size()); |
| Assert.assertTrue(tsFileManager.isAllowCompaction()); |
| } |
| |
| @Test |
| public void testHandleWithAllSourceFilesExistedAndTargetFilesMoved() throws Exception { |
| registerTimeseriesInMManger(4, 5, false); |
| createFiles(2, 2, 3, 300, 0, 0, 50, 50, false, true); |
| createFiles(2, 4, 5, 300, 700, 700, 50, 50, false, true); |
| createFiles(3, 3, 4, 200, 20, 10020, 30, 30, false, false); |
| createFiles(2, 1, 5, 100, 450, 20450, 0, 0, false, false); |
| TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath()); |
| tsFileManager.addAll(seqResources, true); |
| tsFileManager.addAll(unseqResources, false); |
| |
| List<TsFileResource> targetResources = |
| CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); |
| File compactionLogFile = |
| new File( |
| SEQ_DIRS, |
| targetResources.get(0).getTsFile().getName() |
| + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); |
| CompactionLogger compactionLogger = new CompactionLogger(compactionLogFile); |
| compactionLogger.logFiles(targetResources, STR_TARGET_FILES); |
| compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); |
| compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); |
| ICompactionPerformer performer = |
| new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources); |
| performer.setSummary(new CompactionTaskSummary()); |
| performer.perform(); |
| compactionLogger.close(); |
| CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); |
| CompactionExceptionHandler.handleException( |
| COMPACTION_TEST_SG, |
| compactionLogFile, |
| targetResources, |
| seqResources, |
| unseqResources, |
| tsFileManager, |
| 0, |
| false, |
| true); |
| // all source file should still exist |
| for (TsFileResource resource : seqResources) { |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| for (TsFileResource resource : unseqResources) { |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| // tmp target file, target file and target resource file should be deleted |
| for (TsFileResource resource : targetResources) { |
| Assert.assertFalse(resource.getTsFile().exists()); |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)) |
| .exists()); |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX, |
| TsFileConstant.TSFILE_SUFFIX) |
| + TsFileResource.RESOURCE_SUFFIX) |
| .exists()); |
| } |
| Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); |
| Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size()); |
| Assert.assertTrue(tsFileManager.isAllowCompaction()); |
| } |
| |
| @Test |
| public void testHandleWithSomeSourceFilesExisted() throws Exception { |
| registerTimeseriesInMManger(4, 5, false); |
| createFiles(2, 2, 3, 300, 0, 0, 50, 50, false, true); |
| createFiles(2, 4, 5, 300, 700, 700, 50, 50, false, true); |
| createFiles(3, 3, 4, 200, 20, 10020, 30, 30, false, false); |
| createFiles(2, 1, 5, 100, 450, 20450, 0, 0, false, false); |
| TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath()); |
| |
| tsFileManager.addAll(seqResources, true); |
| tsFileManager.addAll(unseqResources, false); |
| |
| List<TsFileResource> targetResources = |
| CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); |
| File compactionLogFile = |
| new File( |
| SEQ_DIRS, |
| targetResources.get(0).getTsFile().getName() |
| + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); |
| CompactionLogger compactionLogger = new CompactionLogger(compactionLogFile); |
| compactionLogger.logFiles(targetResources, STR_TARGET_FILES); |
| compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); |
| compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); |
| ICompactionPerformer performer = |
| new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources); |
| performer.setSummary(new CompactionTaskSummary()); |
| performer.perform(); |
| CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); |
| tsFileManager.replace(seqResources, unseqResources, targetResources, 0, true); |
| seqResources.get(0).getTsFile().delete(); |
| compactionLogger.close(); |
| CompactionExceptionHandler.handleException( |
| COMPACTION_TEST_SG, |
| compactionLogFile, |
| targetResources, |
| seqResources, |
| unseqResources, |
| tsFileManager, |
| 0, |
| false, |
| true); |
| // all source file should not exist |
| for (TsFileResource resource : seqResources) { |
| Assert.assertFalse(resource.getTsFile().exists()); |
| } |
| // tmp target file and tmp target resource file should not exist, target file and target |
| // resource file should exist |
| for (TsFileResource resource : targetResources) { |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)) |
| .exists()); |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX) |
| + TsFileResource.RESOURCE_SUFFIX) |
| .exists()); |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| List<TsFileResource> resources = new ArrayList<>(); |
| for (TsFileResource resource : tsFileManager.getSequenceListByTimePartition(0)) { |
| resources.add(resource); |
| } |
| |
| Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); |
| Assert.assertEquals(0, tsFileManager.getUnsequenceListByTimePartition(0).size()); |
| Assert.assertTrue(tsFileManager.isAllowCompaction()); |
| } |
| |
| @Test |
| public void testHandleWithSomeSameTimeStampSourceFilesExisted() throws Exception { |
| registerTimeseriesInMManger(4, 5, false); |
| createFiles(2, 2, 3, 300, 0, 0, 50, 50, false, true); |
| createFiles(2, 4, 5, 300, 700, 700, 50, 50, false, true); |
| createFiles(3, 3, 4, 200, 20, 10020, 30, 30, false, false); |
| createFiles(2, 1, 5, 100, 450, 20450, 0, 0, false, false); |
| TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath()); |
| TsFileName tsFileName1 = TsFileName.parse(seqResources.get(1).getTsFile().getName()); |
| TsFileName tsFileName2 = TsFileName.parse(seqResources.get(2).getTsFile().getName()); |
| tsFileName2.setTime(tsFileName1.getTime()); |
| File newTsFile = |
| new File(seqResources.get(2).getTsFile().getParent(), tsFileName2.toFileName()); |
| seqResources.get(2).getTsFile().renameTo(newTsFile); |
| File resourceFile = |
| new File(seqResources.get(2).getTsFilePath() + TsFileResource.RESOURCE_SUFFIX); |
| resourceFile.renameTo( |
| new File( |
| resourceFile.getParent(), tsFileName2.toFileName() + TsFileResource.RESOURCE_SUFFIX)); |
| seqResources.get(2).setFile(newTsFile); |
| |
| tsFileManager.addAll(seqResources, true); |
| tsFileManager.addAll(unseqResources, false); |
| |
| List<TsFileResource> targetResources = |
| CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); |
| File compactionLogFile = |
| new File( |
| SEQ_DIRS, |
| targetResources.get(0).getTsFile().getName() |
| + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); |
| CompactionLogger compactionLogger = new CompactionLogger(compactionLogFile); |
| compactionLogger.logFiles(targetResources, STR_TARGET_FILES); |
| compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); |
| compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); |
| ICompactionPerformer performer = |
| new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources); |
| performer.setSummary(new CompactionTaskSummary()); |
| performer.perform(); |
| CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); |
| tsFileManager.replace(seqResources, unseqResources, targetResources, 0, true); |
| seqResources.get(0).getTsFile().delete(); |
| compactionLogger.close(); |
| CompactionExceptionHandler.handleException( |
| COMPACTION_TEST_SG, |
| compactionLogFile, |
| targetResources, |
| seqResources, |
| unseqResources, |
| tsFileManager, |
| 0, |
| false, |
| true); |
| // all source file should not exist |
| for (TsFileResource resource : seqResources) { |
| Assert.assertFalse(resource.getTsFile().exists()); |
| } |
| // tmp target file and tmp target resource file should not exist, target file and target |
| // resource file should exist |
| for (TsFileResource resource : targetResources) { |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)) |
| .exists()); |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX) |
| + TsFileResource.RESOURCE_SUFFIX) |
| .exists()); |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| List<TsFileResource> resources = new ArrayList<>(); |
| for (TsFileResource resource : tsFileManager.getSequenceListByTimePartition(0)) { |
| resources.add(resource); |
| } |
| |
| Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); |
| Assert.assertEquals(0, tsFileManager.getUnsequenceListByTimePartition(0).size()); |
| Assert.assertTrue(tsFileManager.isAllowCompaction()); |
| } |
| |
| /** |
| * Some source files have been deleted, each source file has old mods file and new compaction mods |
| * file. |
| */ |
| @Test |
| public void testHandleWithoutAllSourceFilesAndModFilesExist() throws Exception { |
| registerTimeseriesInMManger(4, 5, false); |
| createFiles(2, 2, 3, 300, 0, 0, 50, 50, false, true); |
| createFiles(2, 4, 5, 300, 700, 700, 50, 50, false, true); |
| createFiles(3, 3, 4, 200, 20, 10020, 30, 30, false, false); |
| createFiles(2, 1, 5, 100, 450, 20450, 0, 0, false, false); |
| TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath()); |
| tsFileManager.addAll(seqResources, true); |
| tsFileManager.addAll(unseqResources, false); |
| |
| List<TsFileResource> targetResources = |
| CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); |
| File compactionLogFile = |
| new File( |
| SEQ_DIRS, |
| targetResources.get(0).getTsFile().getName() |
| + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); |
| CompactionLogger compactionLogger = new CompactionLogger(compactionLogFile); |
| compactionLogger.logFiles(targetResources, STR_TARGET_FILES); |
| compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); |
| compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); |
| ICompactionPerformer performer = |
| new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources); |
| performer.setSummary(new CompactionTaskSummary()); |
| performer.perform(); |
| CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); |
| compactionLogger.close(); |
| for (int i = 0; i < seqResources.size(); i++) { |
| Map<String, Pair<Long, Long>> deleteMap = new HashMap<>(); |
| deleteMap.put( |
| COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0", |
| new Pair(i * 10L, i * 10L + 10)); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), true); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); |
| } |
| for (int i = 0; i < unseqResources.size(); i++) { |
| Map<String, Pair<Long, Long>> deleteMap = new HashMap<>(); |
| deleteMap.put( |
| COMPACTION_TEST_SG + PATH_SEPARATOR + "d1" + PATH_SEPARATOR + "s1", |
| new Pair(i * 10L, i * 10L + 10)); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), true); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false); |
| } |
| CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources); |
| tsFileManager.replace(seqResources, unseqResources, targetResources, 0, true); |
| seqResources.get(0).remove(); |
| |
| CompactionExceptionHandler.handleException( |
| COMPACTION_TEST_SG, |
| compactionLogFile, |
| targetResources, |
| seqResources, |
| unseqResources, |
| tsFileManager, |
| 0, |
| false, |
| true); |
| // All source file should not exist. All compaction mods file and old mods file of each source |
| // file should not exist |
| for (TsFileResource resource : seqResources) { |
| Assert.assertFalse(resource.getTsFile().exists()); |
| Assert.assertFalse(resource.getCompactionModFile().exists()); |
| Assert.assertFalse(resource.getModFile().exists()); |
| } |
| for (TsFileResource resource : unseqResources) { |
| Assert.assertFalse(resource.getTsFile().exists()); |
| Assert.assertFalse(resource.getCompactionModFile().exists()); |
| Assert.assertFalse(resource.getModFile().exists()); |
| } |
| // tmp target file and tmp target resource file should not exist, target file and target |
| // resource file should exist |
| for (TsFileResource resource : targetResources) { |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)) |
| .exists()); |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX) |
| + TsFileResource.RESOURCE_SUFFIX) |
| .exists()); |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| // mods file of the target file should exist |
| Assert.assertTrue(resource.getModFile().exists()); |
| } |
| |
| // compaction log file should not exist |
| Assert.assertFalse(compactionLogFile.exists()); |
| |
| Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); |
| Assert.assertEquals(0, tsFileManager.getUnsequenceListByTimePartition(0).size()); |
| Assert.assertTrue(tsFileManager.isAllowCompaction()); |
| } |
| |
| /** |
| * All source files exist, each source file has compaction mods file which have been combined into |
| * new mods file of the target file. |
| */ |
| @Test |
| public void testHandleWithAllSourcesFileAndCompactonModFileExist() throws Exception { |
| registerTimeseriesInMManger(4, 5, false); |
| createFiles(2, 2, 3, 300, 0, 0, 50, 50, false, true); |
| createFiles(2, 4, 5, 300, 700, 700, 50, 50, false, true); |
| createFiles(3, 3, 4, 200, 20, 10020, 30, 30, false, false); |
| createFiles(2, 1, 5, 100, 450, 20450, 0, 0, false, false); |
| TsFileManager tsFileManager = new TsFileManager(COMPACTION_TEST_SG, "0", SEQ_DIRS.getPath()); |
| tsFileManager.addAll(seqResources, true); |
| tsFileManager.addAll(unseqResources, false); |
| |
| List<TsFileResource> targetResources = |
| CompactionFileGeneratorUtils.getCrossCompactionTargetTsFileResources(seqResources); |
| File compactionLogFile = |
| new File( |
| SEQ_DIRS, |
| targetResources.get(0).getTsFile().getName() |
| + CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX); |
| CompactionLogger compactionLogger = new CompactionLogger(compactionLogFile); |
| compactionLogger.logFiles(targetResources, STR_TARGET_FILES); |
| compactionLogger.logFiles(seqResources, STR_SOURCE_FILES); |
| compactionLogger.logFiles(unseqResources, STR_SOURCE_FILES); |
| ICompactionPerformer performer = |
| new ReadPointCompactionPerformer(seqResources, unseqResources, targetResources); |
| performer.setSummary(new CompactionTaskSummary()); |
| performer.perform(); |
| CompactionUtils.moveTargetFile(targetResources, false, COMPACTION_TEST_SG); |
| compactionLogger.close(); |
| for (int i = 0; i < seqResources.size(); i++) { |
| Map<String, Pair<Long, Long>> deleteMap = new HashMap<>(); |
| deleteMap.put( |
| COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0", |
| new Pair(i * 10L, i * 10L + 10)); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), true); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, seqResources.get(i), false); |
| } |
| for (int i = 0; i < unseqResources.size(); i++) { |
| Map<String, Pair<Long, Long>> deleteMap = new HashMap<>(); |
| deleteMap.put( |
| COMPACTION_TEST_SG + PATH_SEPARATOR + "d1" + PATH_SEPARATOR + "s1", |
| new Pair(i * 10L, i * 10L + 10)); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), true); |
| CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResources.get(i), false); |
| } |
| CompactionUtils.combineModsInCrossCompaction(seqResources, unseqResources, targetResources); |
| |
| CompactionExceptionHandler.handleException( |
| COMPACTION_TEST_SG, |
| compactionLogFile, |
| targetResources, |
| seqResources, |
| unseqResources, |
| tsFileManager, |
| 0, |
| false, |
| true); |
| // all source file should still exist |
| for (TsFileResource resource : seqResources) { |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| for (TsFileResource resource : unseqResources) { |
| Assert.assertTrue(resource.getTsFile().exists()); |
| Assert.assertTrue( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| } |
| // tmp target file, target file and target resource file should be deleted |
| for (TsFileResource resource : targetResources) { |
| // xxx.tsfile should not exist |
| Assert.assertFalse(resource.getTsFile().exists()); |
| // xxx.merge should not exist |
| Assert.assertFalse( |
| new File( |
| resource |
| .getTsFilePath() |
| .replace( |
| TsFileConstant.TSFILE_SUFFIX, |
| IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX)) |
| .exists()); |
| // xxx.tsfile.resource should not exist |
| Assert.assertFalse( |
| new File(resource.getTsFilePath() + TsFileResource.RESOURCE_SUFFIX).exists()); |
| |
| // mods file of the target file should not exist |
| Assert.assertFalse(resource.getModFile().exists()); |
| } |
| |
| // all compaction mods file of each source file should not exist |
| for (int i = 0; i < seqResources.size(); i++) { |
| seqResources.get(i).resetModFile(); |
| ModificationFile f = seqResources.get(i).getCompactionModFile(); |
| Assert.assertFalse(f.exists()); |
| } |
| for (int i = 0; i < unseqResources.size(); i++) { |
| unseqResources.get(i).resetModFile(); |
| Assert.assertFalse(unseqResources.get(i).getCompactionModFile().exists()); |
| } |
| |
| // all mods file of each source file should exist |
| for (TsFileResource resource : seqResources) { |
| resource.resetModFile(); |
| Assert.assertTrue(resource.getModFile().exists()); |
| Assert.assertEquals(1, resource.getModFile().getModifications().size()); |
| } |
| for (TsFileResource resource : unseqResources) { |
| resource.resetModFile(); |
| Assert.assertTrue(resource.getModFile().exists()); |
| Assert.assertEquals(1, resource.getModFile().getModifications().size()); |
| } |
| |
| // compaction log file should not exist |
| Assert.assertFalse(compactionLogFile.exists()); |
| |
| Assert.assertEquals(4, tsFileManager.getSequenceListByTimePartition(0).size()); |
| Assert.assertEquals(5, tsFileManager.getUnsequenceListByTimePartition(0).size()); |
| Assert.assertTrue(tsFileManager.isAllowCompaction()); |
| } |
| } |
