[IOTDB-3656] mpp load with supporting modify MAX_PLAN_NODE_SIZE (#7367)
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 877bbfa..e9892da 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -99,6 +99,10 @@
# Datatype: int
# connection_timeout_ms=20000
+# max plan node size, note that thrift max frame size must be larger than max plan node size.
+# Datatype: int
+# max_plan_node_size=536870912
+
# The maximum number of clients that can be idle for a node's InternalService.
# When the number of idle clients on a node exceeds this number, newly returned clients will be released
# Datatype: int
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 8409527..df25bd8 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -759,9 +759,6 @@
*/
private long partitionInterval = 86400;
- /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */
- private long maxPlanNodeSize = 500 * 1048576L;
-
/**
* Level of TimeIndex, which records the start time and end time of TsFileResource. Currently,
* DEVICE_TIME_INDEX and FILE_TIME_INDEX are supported, and could not be changed after first set.
@@ -813,6 +810,9 @@
/** Unit: byte */
private int thriftMaxFrameSize = 536870912;
+ /** Max size of a {@link PlanNode}, mainly used to control memory of {@link LoadTsFileNode}. */
+ private int maxPlanNodeSize = thriftMaxFrameSize;
+
private int thriftDefaultBufferSize = RpcUtils.THRIFT_DEFAULT_BUF_CAPACITY;
/** time interval in minute for calculating query frequency. Unit: minute */
@@ -2445,6 +2445,10 @@
RpcTransportFactory.setThriftMaxFrameSize(this.thriftMaxFrameSize);
}
+ public void setMaxPlanNodeSize(int maxPlanNodeSize) {
+ this.maxPlanNodeSize = maxPlanNodeSize;
+ }
+
public int getThriftDefaultBufferSize() {
return thriftDefaultBufferSize;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 54a7d4b..1aae7bf 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -773,6 +773,21 @@
conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2);
}
+ conf.setMaxPlanNodeSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_plan_node_size", String.valueOf(conf.getThriftMaxFrameSize()))));
+
+ if (conf.getMaxPlanNodeSize() > conf.getThriftMaxFrameSize()) {
+ logger.warn(
+ String.format(
+ "MAX PLAN NODE SIZE %d can not be larger than THRIFT MAX FRAME SIZE %d, MAX PLAN NODE SIZE has been reset to %d",
+ conf.getMaxPlanNodeSize(),
+ conf.getThriftMaxFrameSize(),
+ conf.getThriftMaxFrameSize()));
+ conf.setMaxPlanNodeSize(conf.getThriftMaxFrameSize());
+ }
+
conf.setThriftDefaultBufferSize(
Integer.parseInt(
properties.getProperty(
