fix redundant operations in compaction
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 54b21dd..629493f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
@@ -193,13 +193,6 @@ ttlDeletion = CompactionUtils.convertTtlToDeletion( device, deviceIterator.getTimeLowerBoundForCurrentDevice()); - for (TsFileResource sourceFile : sortedSourceFiles) { - modificationCache - .computeIfAbsent( - sourceFile.getTsFile().getName(), - k -> PatternTreeMapFactory.getModsPatternTreeMap()) - .append(ttlDeletion.keyOfPatternTree(), ttlDeletion); - } } compactionWriter.setTTLDeletion(ttlDeletion);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 9099586..3129677 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -294,79 +294,9 @@ * Get all measurements and their timeseries metadata offset in each source file. It is used for * new fast compaction to compact nonAligned timeseries. * - * @return measurement -> tsfile resource -> timeseries metadata <startOffset, endOffset> + * @return measurement -> CompactionSeriesContext * @throws IOException if io errors occurred */ - public Map<String, Map<TsFileResource, Pair<Long, Long>>> - getTimeseriesMetadataOffsetOfCurrentDevice() throws IOException { - Map<String, Map<TsFileResource, Pair<Long, Long>>> timeseriesMetadataOffsetMap = - new HashMap<>(); - Map<String, TSDataType> measurementDataTypeMap = new HashMap<>(); - for (TsFileResource resource : tsFileResourcesSortedByDesc) { - if (!deviceIteratorMap.containsKey(resource) - || !deviceIteratorMap.get(resource).current().equals(currentDevice)) { - // if this tsfile has no more device or next device is not equals to the current device, - // which means this tsfile does not contain the current device, then skip it. - continue; - } - TsFileSequenceReader reader = readerMap.get(resource); - for (Map.Entry<String, Pair<TimeseriesMetadata, Pair<Long, Long>>> entrySet : - ((CompactionTsFileReader) reader) - .getTimeseriesMetadataAndOffsetByDevice( - deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), - Collections.emptySet(), - false) - .entrySet()) { - String measurementId = entrySet.getKey(); - // skip the TimeseriesMetadata whose data type is not consistent - TSDataType dataTypeOfCurrentTimeseriesMetadata = entrySet.getValue().left.getTsDataType(); - TSDataType correctDataTypeOfCurrentMeasurement = - measurementDataTypeMap.putIfAbsent(measurementId, dataTypeOfCurrentTimeseriesMetadata); - if (correctDataTypeOfCurrentMeasurement != null - && !MetadataUtils.canAlter( - dataTypeOfCurrentTimeseriesMetadata, correctDataTypeOfCurrentMeasurement)) { - continue; - } - timeseriesMetadataOffsetMap.putIfAbsent(measurementId, new HashMap<>()); - timeseriesMetadataOffsetMap.get(measurementId).put(resource, entrySet.getValue().right); - } - } - return timeseriesMetadataOffsetMap; - } - - /** - * Get all measurement data types of the current device from source files. Traverse all the files - * from the newest to the oldest in turn and start traversing the index tree from the - * firstMeasurementNode node to get all the measurement types under the current device. - * - * @return measurement -> data type - * @throws IOException if io errors occurred - */ - public Map<String, TSDataType> getDataTypeOfCurrentDevice() throws IOException { - Map<String, TSDataType> measurementNameDataTypeMap = new HashMap<>(); - for (TsFileResource resource : tsFileResourcesSortedByDesc) { - if (!deviceIteratorMap.containsKey(resource) - || !deviceIteratorMap.get(resource).current().equals(currentDevice)) { - // if this tsfile has no more device or next device is not equals to the current device, - // which means this tsfile does not contain the current device, then skip it. - continue; - } - TsFileSequenceReader reader = readerMap.get(resource); - for (Map.Entry<String, Pair<TimeseriesMetadata, Pair<Long, Long>>> entrySet : - ((CompactionTsFileReader) reader) - .getTimeseriesMetadataAndOffsetByDevice( - deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), - Collections.emptySet(), - false) - .entrySet()) { - String measurementId = entrySet.getKey(); - TSDataType dataType = entrySet.getValue().left.getTsDataType(); - measurementNameDataTypeMap.putIfAbsent(measurementId, dataType); - } - } - return measurementNameDataTypeMap; - } - public Map<String, CompactionSeriesContext> getCompactionSeriesContextOfCurrentDevice() throws IOException { Map<String, CompactionSeriesContext> compactionSeriesContextMap = new HashMap<>(); @@ -388,16 +318,21 @@ String measurementId = entrySet.getKey(); TimeseriesMetadata timeseriesMetadata = entrySet.getValue().left; Pair<Long, Long> offset = entrySet.getValue().right; - TSDataType dataType = entrySet.getValue().left.getTsDataType(); - if (compactionSeriesContextMap.get(measurementId) != null - && compactionSeriesContextMap.get(measurementId).getFinalType() != null - && !MetadataUtils.canAlter( - dataType, compactionSeriesContextMap.get(measurementId).getFinalType())) { + TSDataType dataType = timeseriesMetadata.getTsDataType(); + + CompactionSeriesContext compactionSeriesContext = + compactionSeriesContextMap.get(measurementId); + if (compactionSeriesContext != null + && compactionSeriesContext.getFinalType() != null + && !MetadataUtils.canAlter(dataType, compactionSeriesContext.getFinalType())) { continue; } - compactionSeriesContextMap.putIfAbsent(measurementId, new CompactionSeriesContext()); - compactionSeriesContextMap.get(measurementId).put(resource, entrySet.getValue().right); - compactionSeriesContextMap.get(measurementId).setFinalTypeIfAbsent(dataType); + + compactionSeriesContext = + compactionSeriesContextMap.computeIfAbsent( + measurementId, k -> new CompactionSeriesContext()); + compactionSeriesContext.put(resource, offset); + compactionSeriesContext.setFinalTypeIfAbsent(dataType); } } return compactionSeriesContextMap;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index 121d4ca..cf9ed65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
@@ -276,21 +276,27 @@ private boolean isValueChunkDataTypeMatchSchema( List<IChunkMetadata> chunkMetadataListOfOneValueColumn) { - boolean isMatch = false; + boolean needAlter = false; for (IChunkMetadata chunkMetadata : chunkMetadataListOfOneValueColumn) { if (chunkMetadata == null) { continue; } String measurement = chunkMetadata.getMeasurementUid(); IMeasurementSchema schema = measurementSchemaMap.get(measurement); - if (MetadataUtils.canAlter(chunkMetadata.getDataType(), schema.getType())) { - if (schema.getType() != chunkMetadata.getDataType()) { - chunkMetadata.setNewType(schema.getType()); + if (!needAlter) { + // Since all chunks in chunkMetadataListOfOneValueColumn share the same dataType, perform + // the compatibility check and early-return only on the first non-null chunk. + if (!MetadataUtils.canAlter(chunkMetadata.getDataType(), schema.getType())) { + return false; } - isMatch = true; + if (schema.getType() == chunkMetadata.getDataType()) { + return true; + } } + needAlter = true; + chunkMetadata.setNewType(schema.getType()); } - return isMatch; + return true; } /**