Add unit test for caculating time partition slot (#7457)
diff --git a/confignode/src/assembly/resources/conf/iotdb-confignode.properties b/confignode/src/assembly/resources/conf/iotdb-confignode.properties
index 4206734..29be491 100644
--- a/confignode/src/assembly/resources/conf/iotdb-confignode.properties
+++ b/confignode/src/assembly/resources/conf/iotdb-confignode.properties
@@ -131,9 +131,9 @@
# default_ttl=36000000
-# Time partition interval in seconds, default is equal to one day
+# Time partition interval in milliseconds, default is equal to one day
# Datatype: long
-# time_partition_interval_for_routing=86400
+# time_partition_interval_for_routing=86400000
# Default number of SchemaRegion replicas
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index af1b577..56988a9 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -104,8 +104,8 @@
private String temporaryLibDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.UDF_TMP_FOLDER_NAME;
- /** Time partition interval in seconds */
- private long timePartitionInterval = 86400;
+ /** Time partition interval in milliseconds */
+ private long timePartitionInterval = 86400000;
/** Default number of SchemaRegion replicas */
private int schemaReplicationFactor = 1;
diff --git a/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java b/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java
index 5ff8fc8..471a9b5 100644
--- a/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java
+++ b/confignode/src/test/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessorTest.java
@@ -195,7 +195,7 @@
Assert.assertEquals(Long.MAX_VALUE, storageGroupSchema.getTTL());
Assert.assertEquals(1, storageGroupSchema.getSchemaReplicationFactor());
Assert.assertEquals(1, storageGroupSchema.getDataReplicationFactor());
- Assert.assertEquals(86400, storageGroupSchema.getTimePartitionInterval());
+ Assert.assertEquals(86400000, storageGroupSchema.getTimePartitionInterval());
storageGroupSchema = schemaMap.get(sg1);
Assert.assertNotNull(storageGroupSchema);
Assert.assertEquals(sg1, storageGroupSchema.getName());
diff --git a/docs/UserGuide/Data-Concept/Time-Partition.md b/docs/UserGuide/Data-Concept/Time-Partition.md
index aeb2ae3..37b1ca2 100644
--- a/docs/UserGuide/Data-Concept/Time-Partition.md
+++ b/docs/UserGuide/Data-Concept/Time-Partition.md
@@ -23,7 +23,7 @@
## Features
-Time partition divides data according to time, and a time partition is used to save all data within a certain time range. The time partition number is represented by a natural number. Number 0 means January 1, 1970, it will increase by one every partition_interval seconds. Time partition number's calculation formula is timestamp / partition_interval. The main configuration items are as follows:
+Time partition divides data according to time, and a time partition is used to save all data within a certain time range. The time partition number is represented by a natural number. Number 0 means January 1, 1970, it will increase by one every partition_interval milliseconds. Time partition number's calculation formula is timestamp / partition_interval. The main configuration items are as follows:
* enable\_partition
@@ -40,12 +40,12 @@
|:---:|:-------------------------------------------------------------------------------------------------------|
|Description| Time range for dividing storage group, time series data will be divided into groups by this time range |
|Type| Int64 |
-|Default| 86400 |
+|Default| 86400000 |
|Effective| Only allowed to be modified in first start up |
## Configuration example
-Enable time partition and set partition_interval to 86400 (one day), then the data distribution is shown as the following figure:
+Enable time partition and set partition_interval to 86400000 (one day), then the data distribution is shown as the following figure:
<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" src="https://github.com/apache/iotdb-bin-resources/blob/main/docs/UserGuide/Data%20Concept/Time-Partition/time_partition_example.png?raw=true" alt="time partition example">
diff --git a/docs/UserGuide/Reference/ConfigNode-Config-Manual.md b/docs/UserGuide/Reference/ConfigNode-Config-Manual.md
index 61aec50..e878581 100644
--- a/docs/UserGuide/Reference/ConfigNode-Config-Manual.md
+++ b/docs/UserGuide/Reference/ConfigNode-Config-Manual.md
@@ -250,7 +250,8 @@
|:---:|:--------------------------------------------------------------|
|Description| Time partition interval of data when ConfigNode allocate data |
|Type| Long |
-|Default| 86400 |
+|Unit| ms |
+|Default| 86400000 |
|Effective| Only allowed to be modified in first start up |
diff --git a/docs/UserGuide/Reference/DataNode-Config-Manual.md b/docs/UserGuide/Reference/DataNode-Config-Manual.md
index bdfe122..e671ec8 100644
--- a/docs/UserGuide/Reference/DataNode-Config-Manual.md
+++ b/docs/UserGuide/Reference/DataNode-Config-Manual.md
@@ -547,7 +547,8 @@
|:---:|:-------------------------------------------------------------------------------------------------------|
|Description| Time range for dividing storage group, time series data will be divided into groups by this time range |
|Type| Int64 |
-|Default| 86400 |
+|Unit| ms |
+|Default| 86400000 |
|Effective| Only allowed to be modified in first start up |
diff --git a/docs/zh/UserGuide/Data-Concept/Time-Partition.md b/docs/zh/UserGuide/Data-Concept/Time-Partition.md
index ea481e6..807284b 100644
--- a/docs/zh/UserGuide/Data-Concept/Time-Partition.md
+++ b/docs/zh/UserGuide/Data-Concept/Time-Partition.md
@@ -23,7 +23,7 @@
## 主要功能
-时间分区按照时间分割数据，一个时间分区用于保存某个时间范围内的所有数据。时间分区编号使用自然数表示，0 表示 1970 年 1 月 1 日，每隔 partition_interval 秒后加一。数据通过计算 timestamp / partition_interval 得到自己所在的时间分区编号，主要配置项如下所示：
+时间分区按照时间分割数据，一个时间分区用于保存某个时间范围内的所有数据。时间分区编号使用自然数表示，0 表示 1970 年 1 月 1 日，每隔 partition_interval 毫秒后加一。数据通过计算 timestamp / partition_interval 得到自己所在的时间分区编号，主要配置项如下所示：
* enable\_partition
@@ -38,14 +38,14 @@
|名字| time\_partition\_interval\_for\_storage |
|:---:|:----------------------------------------|
-|描述| 存储组分区的时间段长度，用户指定的存储组下会使用该时间段进行分区，单位：秒 |
+|描述| 存储组分区的时间段长度，用户指定的存储组下会使用该时间段进行分区，单位：毫秒 |
|类型| Int64 |
-|默认值| 86400 |
+|默认值| 86400000 |
|改后生效方式| 仅允许在第一次启动服务前修改 |
## 配置示例
-开启时间分区功能，并设置 partition_interval 为 86400（一天），则数据的分布情况如下图所示：
+开启时间分区功能，并设置 partition_interval 为 86400000（一天），则数据的分布情况如下图所示：
<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" src="https://github.com/apache/iotdb-bin-resources/blob/main/docs/UserGuide/Data%20Concept/Time-Partition/time_partition_example.png?raw=true" alt="time partition example">
diff --git a/docs/zh/UserGuide/Reference/ConfigNode-Config-Manual.md b/docs/zh/UserGuide/Reference/ConfigNode-Config-Manual.md
index 45b1dbc..43cec10 100644
--- a/docs/zh/UserGuide/Reference/ConfigNode-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/ConfigNode-Config-Manual.md
@@ -242,10 +242,11 @@
* time\_partition\_interval\_for\_routing
|名字| time\_partition\_interval\_for\_routing |
-|:---:|:----------------------------------------|
+|:--:|:----------------------------------------|
|描述| 存储组默认的数据时间分区间隔 |
|类型| Long |
-|默认值| 86400 |
+|单位| 毫秒 |
+|默认值| 86400000 |
|改后生效方式| 仅允许在第一次启动服务前修改 |
### 数据目录
diff --git a/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md b/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md
index 2c389d8..bd647c4 100644
--- a/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/DataNode-Config-Manual.md
@@ -1658,12 +1658,12 @@
* time\_partition\_interval\_for\_storage
-|名字| time\_partition\_interval\_for\_storage |
-|:---:|:----------------------------------------|
-|描述| 用于存储组分区的时间段长度，用户指定的存储组下会使用该时间段进行分区，单位：秒 |
-|类型| Int64 |
-|默认值| 86400 |
-|改后生效方式| 仅允许在第一次启动服务前修改 |
+|名字| time\_partition\_interval\_for\_storage |
+|:---:|:-----------------------------------------|
+|描述| 用于存储组分区的时间段长度，用户指定的存储组下会使用该时间段进行分区，单位：毫秒 |
+|类型| Int64 |
+|默认值| 86400000 |
+|改后生效方式| 仅允许在第一次启动服务前修改 |
* data\_region\_num
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/IoTDBClusterPartitionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/IoTDBClusterPartitionIT.java
index d64d20c..3eda506 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/IoTDBClusterPartitionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/IoTDBClusterPartitionIT.java
@@ -81,7 +81,7 @@
private static final int testReplicationFactor = 3;
protected static long originalTimePartitionInterval;
- private static final long testTimePartitionInterval = 86400;
+ private static final long testTimePartitionInterval = 86400000;
private static final String sg = "root.sg";
private static final int storageGroupNum = 5;
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 38d0e3d..067e1ed 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -1061,9 +1061,9 @@
# Datatype: long
# partition_interval=86400
-# time range for partitioning data inside each data region, the unit is second
+# time range for partitioning data inside each data region, the unit is millisecond
# Datatype: long
-# time_partition_interval_for_storage=86400
+# time_partition_interval_for_storage=86400000
####################
### Influx DB RPC Service Configuration
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 035a4dc..e47c3ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -767,11 +767,11 @@
*/
private long partitionInterval = 86400;
- /** Time partition interval for storage in seconds */
- private long timePartitionIntervalForStorage = 86400;
+ /** Time partition interval for storage in milliseconds */
+ private long timePartitionIntervalForStorage = 86400000;
- /** Time partition interval for routing in seconds */
- private long timePartitionIntervalForRouting = 86400;
+ /** Time partition interval for routing in milliseconds */
+ private long timePartitionIntervalForRouting = 86400000;
/**
* Level of TimeIndex, which records the start time and end time of TsFileResource. Currently,
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 3ee6ee8..8ee67e2 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1013,6 +1013,9 @@
// author cache
loadAuthorCache(properties);
+
+ conf.setTimePartitionIntervalForStorage(
+ convertMilliWithPrecision(conf.getTimePartitionIntervalForStorage()));
}
private void loadAuthorCache(Properties properties) {
@@ -1921,7 +1924,8 @@
public void loadGlobalConfig(TGlobalConfig globalConfig) {
conf.setSeriesPartitionExecutorClass(globalConfig.getSeriesPartitionExecutorClass());
conf.setSeriesPartitionSlotNum(globalConfig.getSeriesPartitionSlotNum());
- conf.setTimePartitionIntervalForRouting(globalConfig.timePartitionInterval);
+ conf.setTimePartitionIntervalForRouting(
+ convertMilliWithPrecision(globalConfig.timePartitionInterval));
conf.setReadConsistencyLevel(globalConfig.getReadConsistencyLevel());
}
@@ -1988,6 +1992,22 @@
logger.info("Cluster allocateMemoryForLastCache = {}", conf.getAllocateMemoryForLastCache());
}
+ public long convertMilliWithPrecision(long milliTime) {
+ long result = milliTime;
+ String timePrecision = conf.getTimestampPrecision();
+ switch (timePrecision) {
+ case "ns":
+ result = milliTime * 1000_000L;
+ break;
+ case "us":
+ result = milliTime * 1000L;
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+
private static class IoTDBDescriptorHolder {
private static final IoTDBDescriptor INSTANCE = new IoTDBDescriptor();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
index 6bfe588..3ab8bc6 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
@@ -55,7 +55,6 @@
import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.db.sync.SyncService;
import org.apache.iotdb.db.utils.ThreadUtils;
-import org.apache.iotdb.db.utils.TimePartitionUtils;
import org.apache.iotdb.db.utils.UpgradeUtils;
import org.apache.iotdb.db.wal.WALManager;
import org.apache.iotdb.db.wal.exception.WALException;
@@ -148,8 +147,7 @@
private static void initTimePartition() {
timePartitionIntervalForStorage =
- TimePartitionUtils.convertMilliWithPrecision(
- IoTDBDescriptor.getInstance().getConfig().getTimePartitionIntervalForStorage() * 1000L);
+ IoTDBDescriptor.getInstance().getConfig().getTimePartitionIntervalForStorage();
}
public static long getTimePartitionIntervalForStorage() {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakePartitionFetcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakePartitionFetcherImpl.java
index 014bc6e..7601211 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakePartitionFetcherImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/FakePartitionFetcherImpl.java
@@ -30,6 +30,7 @@
import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition;
import org.apache.iotdb.commons.partition.SchemaPartition;
+import org.apache.iotdb.commons.partition.executor.SeriesPartitionExecutor;
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq;
@@ -42,6 +43,16 @@
public class FakePartitionFetcherImpl implements IPartitionFetcher {
+ private final String seriesSlotExecutorName =
+ IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass();
+
+ private final int seriesPartitionSlotNum =
+ IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionSlotNum();
+
+ private final SeriesPartitionExecutor partitionExecutor =
+ SeriesPartitionExecutor.getSeriesPartitionExecutor(
+ seriesSlotExecutorName, seriesPartitionSlotNum);
+
@Override
public SchemaPartition getSchemaPartition(PathPatternTree patternTree) {
String device1 = "root.sg.d1";
@@ -212,7 +223,71 @@
@Override
public DataPartition getOrCreateDataPartition(
List<DataPartitionQueryParam> dataPartitionQueryParams) {
- return null;
+
+ // only test root.sg
+ DataPartition dataPartition =
+ new DataPartition(
+ IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionExecutorClass(),
+ IoTDBDescriptor.getInstance().getConfig().getSeriesPartitionSlotNum());
+
+ Map<String, Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>>>
+ dataPartitionMap = new HashMap<>();
+ Map<TSeriesPartitionSlot, Map<TTimePartitionSlot, List<TRegionReplicaSet>>> sgPartitionMap =
+ new HashMap<>();
+ dataPartitionMap.put("root.sg", sgPartitionMap);
+ dataPartition.setDataPartitionMap(dataPartitionMap);
+
+ List<TRegionReplicaSet> d1DataRegions = new ArrayList<>();
+ d1DataRegions.add(
+ new TRegionReplicaSet(
+ new TConsensusGroupId(TConsensusGroupType.DataRegion, 1),
+ Arrays.asList(
+ new TDataNodeLocation()
+ .setDataNodeId(11)
+ .setClientRpcEndPoint(new TEndPoint("192.0.1.1", 9000)),
+ new TDataNodeLocation()
+ .setDataNodeId(12)
+ .setClientRpcEndPoint(new TEndPoint("192.0.1.2", 9000)))));
+ d1DataRegions.add(
+ new TRegionReplicaSet(
+ new TConsensusGroupId(TConsensusGroupType.DataRegion, 2),
+ Arrays.asList(
+ new TDataNodeLocation()
+ .setDataNodeId(21)
+ .setClientRpcEndPoint(new TEndPoint("192.0.2.1", 9000)),
+ new TDataNodeLocation()
+ .setDataNodeId(22)
+ .setClientRpcEndPoint(new TEndPoint("192.0.2.2", 9000)))));
+
+ List<TRegionReplicaSet> d2DataRegions = new ArrayList<>();
+ d2DataRegions.add(
+ new TRegionReplicaSet(
+ new TConsensusGroupId(TConsensusGroupType.DataRegion, 3),
+ Arrays.asList(
+ new TDataNodeLocation()
+ .setDataNodeId(31)
+ .setClientRpcEndPoint(new TEndPoint("192.0.3.1", 9000)),
+ new TDataNodeLocation()
+ .setDataNodeId(32)
+ .setClientRpcEndPoint(new TEndPoint("192.0.3.2", 9000)))));
+ Map<TTimePartitionSlot, List<TRegionReplicaSet>> d2DataRegionMap = new HashMap<>();
+ d2DataRegionMap.put(new TTimePartitionSlot(), d2DataRegions);
+
+ for (DataPartitionQueryParam dataPartitionQueryParam : dataPartitionQueryParams) {
+ TSeriesPartitionSlot seriesPartitionSlot =
+ partitionExecutor.getSeriesPartitionSlot(dataPartitionQueryParam.getDevicePath());
+ Map<TTimePartitionSlot, List<TRegionReplicaSet>> timePartitionSlotListMap =
+ sgPartitionMap.computeIfAbsent(seriesPartitionSlot, k -> new HashMap<>());
+ for (TTimePartitionSlot timePartitionSlot :
+ dataPartitionQueryParam.getTimePartitionSlotList()) {
+ if (timePartitionSlot.startTime == 0) {
+ timePartitionSlotListMap.put(timePartitionSlot, d1DataRegions);
+ } else {
+ timePartitionSlotListMap.put(timePartitionSlot, d2DataRegions);
+ }
+ }
+ }
+ return dataPartition;
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java
index 3376699..f39de5b 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java
@@ -26,8 +26,7 @@
public class TimePartitionUtils {
@ServerConfigConsistent
public static long timePartitionIntervalForRouting =
- convertMilliWithPrecision(
- IoTDBDescriptor.getInstance().getConfig().getTimePartitionIntervalForRouting() * 1000L);
+ IoTDBDescriptor.getInstance().getConfig().getTimePartitionIntervalForRouting();
public static TTimePartitionSlot getTimePartitionForRouting(long time) {
TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot();
@@ -35,22 +34,6 @@
return timePartitionSlot;
}
- public static long convertMilliWithPrecision(long milliTime) {
- long result = milliTime;
- String timePrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
- switch (timePrecision) {
- case "ns":
- result = milliTime * 1000_000L;
- break;
- case "us":
- result = milliTime * 1000L;
- break;
- default:
- break;
- }
- return result;
- }
-
@TestOnly
public static void setTimePartitionIntervalForRouting(long timePartitionIntervalForRouting) {
TimePartitionUtils.timePartitionIntervalForRouting = timePartitionIntervalForRouting;
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java
index 6f43da9..895c470 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.mpp.plan.analyze;
+import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
@@ -571,6 +572,19 @@
}
}
+ @Test
+ public void testDataPartitionAnalyze() {
+ Analysis analysis = analyzeSQL("insert into root.sg.d1(timestamp,s) values(1,10),(86401,11)");
+ Assert.assertEquals(
+ analysis
+ .getDataPartitionInfo()
+ .getDataPartitionMap()
+ .get("root.sg")
+ .get(new TSeriesPartitionSlot(8923))
+ .size(),
+ 1);
+ }
+
private Analysis analyzeSQL(String sql) {
try {
Statement statement =
