add temp tool TsFileOverlapValidationTool (#10576)
diff --git a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/validate-overlap.bat b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/validate-overlap.bat new file mode 100644 index 0000000..71321bf --- /dev/null +++ b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/validate-overlap.bat
@@ -0,0 +1,62 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM + + +@echo off +echo ```````````````````````` +echo Starting Validating the TsFile +echo ```````````````````````` + +if "%OS%" == "Windows_NT" setlocal + +pushd %~dp0..\.. +if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD% +popd + +if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.validate.TsFileOverlapValidationTool +if NOT DEFINED JAVA_HOME goto :err + +@REM ----------------------------------------------------------------------------- +@REM ***** CLASSPATH library setting ***** +@REM Ensure that any user defined CLASSPATH variables are not used on startup +set CLASSPATH="%IOTDB_HOME%\lib\*" + +goto okClasspath + +:append +set CLASSPATH=%CLASSPATH%;%1 +goto :eof + +@REM ----------------------------------------------------------------------------- +:okClasspath + +"%JAVA_HOME%\bin\java" -cp "%CLASSPATH%" %MAIN_CLASS% %* + +goto finally + + +:err +echo JAVA_HOME environment variable must be set! +pause + + +@REM ----------------------------------------------------------------------------- +:finally + +ENDLOCAL
diff --git a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/validate-overlap.sh b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/validate-overlap.sh new file mode 100644 index 0000000..be175d9 --- /dev/null +++ b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/validate-overlap.sh
@@ -0,0 +1,51 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +echo --------------------- +echo Starting Validating the TsFile +echo --------------------- + +source "$(dirname "$0")/../../sbin/iotdb-common.sh" +#get_iotdb_include and checkAllVariables is in iotdb-common.sh +VARS=$(get_iotdb_include "$*") +checkAllVariables +export IOTDB_HOME="${IOTDB_HOME}/.." +eval set -- "$VARS" + +if [ -n "$JAVA_HOME" ]; then + for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do + if [ -x "$java" ]; then + JAVA="$java" + break + fi + done +else + JAVA=java +fi + +CLASSPATH="" +for f in ${IOTDB_HOME}/lib/*.jar; do + CLASSPATH=${CLASSPATH}":"$f +done + +MAIN_CLASS=org.apache.iotdb.db.tools.validate.TsFileOverlapValidationTool + +"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@" +exit $?
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationTool.java new file mode 100644 index 0000000..3782a16 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/validate/TsFileOverlapValidationTool.java
@@ -0,0 +1,119 @@ +package org.apache.iotdb.db.tools.validate; + +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.tsfile.utils.Pair; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; + +public class TsFileOverlapValidationTool { + + public static void main(String[] args) throws IOException { + if (args.length == 0) { + return; + } + String absolutePath = args[0]; + validate(absolutePath); + } + + @SuppressWarnings("java:S3776") + private static void validate(String path) throws IOException { + File seqRoot = new File(path); + if (!seqRoot.isDirectory()) { + return; + } + for (File storageGroup : Objects.requireNonNull(seqRoot.listFiles())) { + if (!storageGroup.isDirectory()) { + continue; + } + for (File dataRegion : Objects.requireNonNull(storageGroup.listFiles())) { + if (!dataRegion.isDirectory()) { + continue; + } + for (File timePartition : Objects.requireNonNull(dataRegion.listFiles())) { + if (!timePartition.isDirectory()) { + continue; + } + List<TsFileResource> resources = loadAndSortFile(timePartition.getAbsolutePath()); + validateTsFileResources(resources); + } + } + } + } + + private static List<TsFileResource> loadAndSortFile(String timePartitionPath) throws IOException { + File dir = new File(timePartitionPath); + // get all seq file resources under the time partition dir + List<File> tsFiles = + Arrays.asList( + Objects.requireNonNull( + dir.listFiles(file -> file.getName().endsWith(TSFILE_SUFFIX)))); + // sort the seq files with timestamp + tsFiles.sort( + (f1, f2) -> { + int timeDiff = + Long.compareUnsigned( + Long.parseLong(f1.getName().split("-")[0]), + Long.parseLong(f2.getName().split("-")[0])); + return timeDiff == 0 + ? Long.compareUnsigned( + Long.parseLong(f1.getName().split("-")[1]), + Long.parseLong(f2.getName().split("-")[1])) + : timeDiff; + }); + List<TsFileResource> tsFileResources = new ArrayList<>(); + for (File tsFile : tsFiles) { + TsFileResource resource = + new TsFileResource(SystemFileFactory.INSTANCE.getFile(tsFile.getAbsolutePath())); + resource.deserialize(); + tsFileResources.add(resource); + } + return tsFileResources; + } + + private static Set<TsFileResource> validateTsFileResources(List<TsFileResource> resources) + throws IOException { + // deviceID -> <TsFileResource, last end time> + Map<String, Pair<TsFileResource, Long>> lastEndTimeMap = new HashMap<>(); + Set<TsFileResource> overlapFiles = new HashSet<>(); + for (TsFileResource resource : resources) { + DeviceTimeIndex timeIndex; + if (resource.getTimeIndexType() != 1) { + // if time index is not device time index, then deserialize it from resource file + timeIndex = resource.buildDeviceTimeIndex(); + } else { + timeIndex = (DeviceTimeIndex) resource.getTimeIndex(); + } + Set<String> devices = timeIndex.getDevices(); + for (String device : devices) { + long currentStartTime = timeIndex.getStartTime(device); + long currentEndTime = timeIndex.getEndTime(device); + Pair<TsFileResource, Long> lastDeviceInfo = + lastEndTimeMap.computeIfAbsent(device, x -> new Pair<>(null, Long.MIN_VALUE)); + long lastEndTime = lastDeviceInfo.right; + if (lastEndTime >= currentStartTime) { + overlapFiles.add(resource); + System.out.println( + "Add File " + resource.getTsFile().getAbsolutePath() + " to overlap file list"); + break; + } + lastDeviceInfo.left = resource; + lastDeviceInfo.right = currentEndTime; + lastEndTimeMap.put(device, lastDeviceInfo); + } + } + return overlapFiles; + } +}