[To dev/1.3] Enhance the last query permission && Fixed the rollback version of alter view / table plans && Deleted the unnecessary mods in Tree view deletion (#17465) (#17494)
* [To dev/1.3] Enhance the last query permission && Fixed the rollback version of alter view / table plans && Deleted the unnecessary mods in Tree view deletion (#17465)
* dele
* shop
* f
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java
index ad1b93a..6151327 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.it;
+import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.env.SimpleEnv;
@@ -57,6 +58,7 @@
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Base64;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -513,6 +515,7 @@
selectLast(httpClient);
queryV2(httpClient);
+ selectFastLast(httpClient);
queryGroupByLevelV2(httpClient);
queryRowLimitV2(httpClient);
queryShowChildPathsV2(httpClient);
@@ -923,6 +926,71 @@
}
}
+ @Test
+ public void queryFastLastWithWrongAuthorization() {
+ CloseableHttpResponse response = null;
+
+ TestUtils.executeNonQuery("create user abcd 'strongPassword@1234'");
+ try {
+ final CloseableHttpClient httpClient = HttpClientBuilder.create().build();
+ final HttpPost httpPost = new HttpPost("http://127.0.0.1:" + port + "/rest/v2/fastLastQuery");
+ httpPost.addHeader("Content-type", "application/json; charset=utf-8");
+ httpPost.setHeader("Accept", "application/json");
+ final String authorization = getAuthorization("abcd", "strongPassword@1234");
+ httpPost.setHeader("Authorization", authorization);
+ final String sql = "{\"prefix_paths\":[\"root\",\"sg25\"]}";
+ httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset()));
+ for (int i = 0; i < 30; i++) {
+ try {
+ response = httpClient.execute(httpPost);
+ break;
+ } catch (Exception e) {
+ if (i == 29) {
+ throw e;
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ }
+
+ Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+ String message = EntityUtils.toString(response.getEntity(), "utf-8");
+ ObjectMapper mapper = new ObjectMapper();
+ Map map = mapper.readValue(message, Map.class);
+ List<Long> timestampsResult = (List<Long>) map.get("timestamps");
+ List<Long> expressionsResult = (List<Long>) map.get("expressions");
+ List<List<Object>> valuesResult = (List<List<Object>>) map.get("values");
+ Assert.assertTrue(map.size() > 0);
+ List<Object> expressions =
+ new ArrayList<Object>() {
+ {
+ add("Timeseries");
+ add("Value");
+ add("DataType");
+ }
+ };
+
+ Assert.assertEquals(expressions, expressionsResult);
+ Assert.assertEquals(Collections.emptyList(), timestampsResult);
+ Assert.assertEquals(Collections.emptyList(), valuesResult);
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ } finally {
+ try {
+ if (response != null) {
+ response.close();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+ }
+
public void query(CloseableHttpClient httpClient) {
CloseableHttpResponse response = null;
try {
@@ -1677,6 +1745,98 @@
}
}
+ public void selectFastLast(CloseableHttpClient httpClient) {
+ // Only used in 1D scenarios
+ if (EnvFactory.getEnv().getDataNodeWrapperList().size() > 1) {
+ return;
+ }
+ CloseableHttpResponse response = null;
+ try {
+ HttpPost httpPost = getHttpPost("http://127.0.0.1:" + port + "/rest/v2/fastLastQuery");
+ String sql = "{\"prefix_paths\":[\"root\",\"sg25\"]}";
+ httpPost.setEntity(new StringEntity(sql, Charset.defaultCharset()));
+ response = httpClient.execute(httpPost);
+ HttpEntity responseEntity = response.getEntity();
+ String message = EntityUtils.toString(responseEntity, "utf-8");
+ ObjectMapper mapper = new ObjectMapper();
+ Map map = mapper.readValue(message, Map.class);
+ List<Long> timestampsResult = (List<Long>) map.get("timestamps");
+ List<Long> expressionsResult = (List<Long>) map.get("expressions");
+ List<List<Object>> valuesResult = (List<List<Object>>) map.get("values");
+ Assert.assertTrue(map.size() > 0);
+ List<Object> expressions =
+ new ArrayList<Object>() {
+ {
+ add("Timeseries");
+ add("Value");
+ add("DataType");
+ }
+ };
+ List<Object> timestamps =
+ new ArrayList<Object>() {
+ {
+ add(1635232153960l);
+ add(1635232153960l);
+ add(1635232153960l);
+ add(1635232143960l);
+ add(1635232153960l);
+ add(1635232153960l);
+ }
+ };
+ List<Object> values1 =
+ new ArrayList<Object>() {
+ {
+ add("root.sg25.s3");
+ add("root.sg25.s4");
+ add("root.sg25.s5");
+ add("root.sg25.s6");
+ add("root.sg25.s7");
+ add("root.sg25.s8");
+ }
+ };
+ List<Object> values2 =
+ new ArrayList<Object>() {
+ {
+ add("");
+ add("2");
+ add("1635000012345556");
+ add("1.41");
+ add("false");
+ add("3.5555");
+ }
+ };
+ List<Object> values3 =
+ new ArrayList<Object>() {
+ {
+ add("TEXT");
+ add("INT32");
+ add("INT64");
+ add("FLOAT");
+ add("BOOLEAN");
+ add("DOUBLE");
+ }
+ };
+
+ Assert.assertEquals(expressions, expressionsResult);
+ Assert.assertEquals(timestamps, timestampsResult);
+ Assert.assertEquals(values1, valuesResult.get(0));
+ Assert.assertEquals(values2, valuesResult.get(1));
+ Assert.assertEquals(values3, valuesResult.get(2));
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ } finally {
+ try {
+ if (response != null) {
+ response.close();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+ }
+
public void queryGroupByLevelV2(CloseableHttpClient httpClient) {
CloseableHttpResponse response = null;
try {
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java
index abf5936..f45209c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java
@@ -684,6 +684,32 @@
}
}
+ public static void assertResultSetEqual(
+ SessionDataSet actualResultSet,
+ List<String> expectedColumnNames,
+ Set<String> expectedRetSet,
+ boolean ignoreTimeStamp) {
+ final Set<String> copiedSet = new HashSet<>(expectedRetSet);
+ try {
+ List<String> actualColumnNames = actualResultSet.getColumnNames();
+ if (ignoreTimeStamp) {
+ assertEquals(expectedColumnNames, actualColumnNames);
+ } else {
+ assertEquals(TIMESTAMP_STR, actualColumnNames.get(0));
+ assertEquals(expectedColumnNames, actualColumnNames.subList(1, actualColumnNames.size()));
+ }
+
+ while (actualResultSet.hasNext()) {
+ RowRecord rowRecord = actualResultSet.next();
+ assertTrue(copiedSet.remove(rowRecord.toString().replace('\t', ',')));
+ }
+ assertEquals(0, copiedSet.size());
+ } catch (IoTDBConnectionException | StatementExecutionException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
public static void createUser(String userName, String password) {
createUser(EnvFactory.getEnv(), userName, password);
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java
index ba5c22a..7ade934 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java
@@ -22,6 +22,7 @@
import org.apache.iotdb.common.rpc.thrift.TAggregationType;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.it.utils.AlignedWriteUtil;
+import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
@@ -29,6 +30,7 @@
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.RedirectException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.junit.AfterClass;
@@ -40,7 +42,9 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import static org.apache.iotdb.db.it.utils.TestUtils.assertResultSetEqual;
import static org.junit.Assert.fail;
@@ -244,6 +248,81 @@
}
}
+ @Test
+ public void lastQueryWithPrefixTest() throws IoTDBConnectionException {
+ // Only used in 1D scenarios
+ if (EnvFactory.getEnv().getDataNodeWrapperList().size() > 1) {
+ return;
+ }
+ final Set<String> retArray =
+ new HashSet<>(
+ Arrays.asList(
+ "30,root.sg1.d1.s3,30,INT64",
+ "30,root.sg1.d1.s4,false,BOOLEAN",
+ "40,root.sg1.d1.s5,aligned_test40,TEXT",
+ "23,root.sg1.d1.s1,230000.0,FLOAT",
+ "40,root.sg1.d1.s2,40,INT32"));
+
+ try (final ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ // Push last cache first
+ try (final SessionDataSet resultSet =
+ session.executeFastLastDataQueryForOnePrefixPath(Arrays.asList("root", "sg1", "d1"))) {
+ assertResultSetEqual(resultSet, lastQueryColumnNames, retArray, true);
+ }
+
+ try (final SessionDataSet resultSet =
+ session.executeFastLastDataQueryForOnePrefixPath(Arrays.asList("root", "sg1", "d1"))) {
+ assertResultSetEqual(resultSet, lastQueryColumnNames, retArray, true);
+ }
+ } catch (StatementExecutionException | RedirectException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void lastQueryWithoutPermissionTest() throws IoTDBConnectionException {
+ // Only used in 1D scenarios
+ if (EnvFactory.getEnv().getDataNodeWrapperList().size() > 1) {
+ return;
+ }
+ final String[] retArray = new String[] {};
+ final Set<String> retArray2 =
+ new HashSet<>(
+ Arrays.asList(
+ "30,root.sg1.d1.s3,30,INT64",
+ "30,root.sg1.d1.s4,false,BOOLEAN",
+ "40,root.sg1.d1.s5,aligned_test40,TEXT",
+ "23,root.sg1.d1.s1,230000.0,FLOAT",
+ "40,root.sg1.d1.s2,40,INT32"));
+ TestUtils.executeNonQuery("create user abcd 'veryComplexPassword@123'");
+
+ try (final ISession session =
+ EnvFactory.getEnv().getSessionConnection("abcd", "veryComplexPassword@123");
+ final ISession rootSession = EnvFactory.getEnv().getSessionConnection()) {
+ // Push last cache first
+ try (final SessionDataSet resultSet =
+ rootSession.executeFastLastDataQueryForOnePrefixPath(
+ Arrays.asList("root", "sg1", "d1"))) {
+ assertResultSetEqual(resultSet, lastQueryColumnNames, retArray2, true);
+ }
+
+ try (final SessionDataSet resultSet =
+ session.executeLastDataQueryForOneDevice(
+ "root.sg1", "root.sg1.d1", Arrays.asList("notExist", "s1"), true)) {
+ assertResultSetEqual(resultSet, lastQueryColumnNames, retArray, true);
+ }
+
+ try (final SessionDataSet resultSet =
+ session.executeFastLastDataQueryForOnePrefixPath(Arrays.asList("root", "sg1", "d1"))) {
+ assertResultSetEqual(resultSet, lastQueryColumnNames, retArray, true);
+ }
+ } catch (StatementExecutionException | RedirectException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
// ------------------------------ Aggregation Query ------------------------------
@Test
public void aggregationQueryTest() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java
index 05fc4b8..0030980 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/impl/RestApiServiceImpl.java
@@ -55,6 +55,7 @@
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
import org.apache.iotdb.db.utils.CommonUtils;
@@ -102,7 +103,7 @@
public Response executeFastLastQueryStatement(
PrefixPathList prefixPathList, SecurityContext securityContext) {
Long queryId = null;
- Statement statement = null;
+ QueryStatement statement = null;
boolean finish = false;
long startTime = System.nanoTime();
@@ -113,28 +114,31 @@
new PartialPath(prefixPathList.getPrefixPaths().toArray(new String[0]));
final Map<PartialPath, Map<String, TimeValuePair>> resultMap = new HashMap<>();
+ // Check permission, the cost is rather low because the req only contains one prefix path
+ final IClientSession clientSession = SESSION_MANAGER.getCurrSession();
+ final TSLastDataQueryReq tsLastDataQueryReq =
+ FastLastHandler.createTSLastDataQueryReq(clientSession, prefixPathList);
+ statement = StatementGenerator.createStatement(tsLastDataQueryReq);
+
+ final Response response = authorizationHandler.checkAuthority(securityContext, statement);
+ if (response != null) {
+ return response;
+ }
+
final String prefixString = prefixPath.toString();
for (ISchemaRegion region : SchemaEngine.getInstance().getAllSchemaRegions()) {
if (!prefixString.startsWith(region.getDatabaseFullPath())
&& !region.getDatabaseFullPath().startsWith(prefixString)) {
continue;
}
- region.fillLastQueryMap(prefixPath, resultMap);
+ region.fillLastQueryMap(prefixPath, resultMap, statement.getAuthorityScope());
}
// Check cache first
if (!DataNodeSchemaCache.getInstance().getDeviceSchemaCache().getLastCache(resultMap)) {
- IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime();
- TSLastDataQueryReq tsLastDataQueryReq =
- FastLastHandler.createTSLastDataQueryReq(clientSession, prefixPathList);
- statement = StatementGenerator.createStatement(tsLastDataQueryReq);
-
if (ExecuteStatementHandler.validateStatement(statement)) {
return FastLastHandler.buildErrorResponse(TSStatusCode.EXECUTE_STATEMENT_ERROR);
}
- Optional.ofNullable(authorizationHandler.checkAuthority(securityContext, statement))
- .ifPresent(Response.class::cast);
-
queryId = SESSION_MANAGER.requestQueryId();
SessionInfo sessionInfo = SESSION_MANAGER.getSessionInfo(clientSession);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
index 8a498fc..2ae50d3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java
@@ -37,6 +37,7 @@
import org.apache.iotdb.commons.path.AlignedPath;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.db.audit.AuditLogger;
import org.apache.iotdb.db.auth.AuthorityChecker;
@@ -87,6 +88,7 @@
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateAlignedTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateMultiTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateTimeSeriesStatement;
@@ -819,7 +821,7 @@
try {
final long queryId = SESSION_MANAGER.requestQueryId(clientSession, req.statementId);
- // 1. Map<Device, String[] measurements> ISchemaFetcher.getAllSensors(prefix) ~= 50ms
+ // 1.1 Map<Device, String[] measurements> ISchemaFetcher.getAllSensors(prefix) ~= 50ms
final PartialPath prefixPath = new PartialPath(req.getPrefixes().toArray(new String[0]));
if (prefixPath.hasWildcard()) {
@@ -832,13 +834,20 @@
final Map<PartialPath, Map<String, TimeValuePair>> resultMap = new HashMap<>();
int sensorNum = 0;
+ // 1.2 Check permission, the cost is rather low because the req only contains one prefix path
+ final QueryStatement s = StatementGenerator.createStatement(convert(req));
+ final TSStatus status = AuthorityChecker.checkAuthority(s, clientSession);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return RpcUtils.getTSExecuteStatementResp(status);
+ }
+
final String prefixString = prefixPath.toString();
for (final ISchemaRegion region : SchemaEngine.getInstance().getAllSchemaRegions()) {
if (!prefixString.startsWith(region.getDatabaseFullPath())
&& !region.getDatabaseFullPath().startsWith(prefixString)) {
continue;
}
- sensorNum += region.fillLastQueryMap(prefixPath, resultMap);
+ sensorNum += region.fillLastQueryMap(prefixPath, resultMap, s.getAuthorityScope());
}
// 2.DATA_NODE_SCHEMA_CACHE.getLastCache()
@@ -914,6 +923,13 @@
long startTime = System.nanoTime();
Throwable t = null;
try {
+ // Place the permission check first
+ final QueryStatement s = StatementGenerator.createStatement(convert(req));
+ final TSStatus status = AuthorityChecker.checkAuthority(s, clientSession);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return RpcUtils.getTSExecuteStatementResp(status);
+ }
+
String db;
String deviceId;
PartialPath devicePath;
@@ -969,9 +985,11 @@
regionReplicaSet ->
isSameNode(
regionReplicaSet.dataNodeLocations.get(0).mPPDataExchangeEndPoint));
- int sensorNum = req.sensors.size();
- TsBlockBuilder builder = LastQueryUtil.createTsBlockBuilder(sensorNum);
+ final int sensorNum = req.sensors.size();
+ final TsBlockBuilder builder = LastQueryUtil.createTsBlockBuilder(sensorNum);
boolean allCached = true;
+
+ PathPatternTree queryTree = new PathPatternTree();
for (String sensor : req.sensors) {
PartialPath fullPath;
if (req.isLegalPathNodes()) {
@@ -979,24 +997,32 @@
} else {
fullPath = devicePath.concatNode((new PartialPath(sensor)).getFullPath());
}
- TimeValuePair timeValuePair = DATA_NODE_SCHEMA_CACHE.getLastCache(fullPath);
- if (timeValuePair == null) {
- allCached = false;
- break;
- } else if (timeValuePair.getValue() == null) {
- // there is no data for this sensor
- if (!canUseNullEntry) {
+ queryTree.appendPathPattern(fullPath);
+ }
+ queryTree.constructTree();
+ queryTree = s.getAuthorityScope().intersectWithFullPathPrefixTree(queryTree);
+
+ if (!queryTree.isEmpty()) {
+ for (final MeasurementPath fullPath : queryTree.getAllPathPatterns(true)) {
+ TimeValuePair timeValuePair = DATA_NODE_SCHEMA_CACHE.getLastCache(fullPath);
+ if (timeValuePair == null) {
allCached = false;
break;
+ } else if (timeValuePair.getValue() == null) {
+ // there is no data for this sensor
+ if (!canUseNullEntry) {
+ allCached = false;
+ break;
+ }
+ } else {
+ // we don't consider TTL
+ LastQueryUtil.appendLastValue(
+ builder,
+ timeValuePair.getTimestamp(),
+ new Binary(fullPath.getFullPath(), TSFileConfig.STRING_CHARSET),
+ timeValuePair.getValue().getStringValue(),
+ timeValuePair.getValue().getDataType().name());
}
- } else {
- // we don't consider TTL
- LastQueryUtil.appendLastValue(
- builder,
- timeValuePair.getTimestamp(),
- new Binary(fullPath.getFullPath(), TSFileConfig.STRING_CHARSET),
- timeValuePair.getValue().getStringValue(),
- timeValuePair.getValue().getDataType().name());
}
}
// cache hit
@@ -1016,14 +1042,6 @@
}
}
- // cache miss
- Statement s = StatementGenerator.createStatement(convert(req));
- // permission check
- TSStatus status = AuthorityChecker.checkAuthority(s, clientSession);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- return RpcUtils.getTSExecuteStatementResp(status);
- }
-
quota =
DataNodeThrottleQuotaManager.getInstance()
.checkQuota(SESSION_MANAGER.getCurrSession().getUsername(), s);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
index b220309..9f149ab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
@@ -170,7 +170,7 @@
return queryStatement;
}
- public static Statement createStatement(TSLastDataQueryReq lastDataQueryReq)
+ public static QueryStatement createStatement(TSLastDataQueryReq lastDataQueryReq)
throws IllegalPathException {
final long startTime = System.nanoTime();
// construct query statement
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
index df69657..7248fe2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
@@ -317,7 +317,9 @@
throws MetadataException;
int fillLastQueryMap(
- final PartialPath pattern, final Map<PartialPath, Map<String, TimeValuePair>> mapToFill)
+ final PartialPath pattern,
+ final Map<PartialPath, Map<String, TimeValuePair>> mapToFill,
+ final PathPatternTree scope)
throws MetadataException;
// endregion
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
index 7f8074c..6fc556a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
@@ -1335,9 +1335,11 @@
@Override
public int fillLastQueryMap(
- final PartialPath pattern, final Map<PartialPath, Map<String, TimeValuePair>> mapToFill)
+ final PartialPath pattern,
+ final Map<PartialPath, Map<String, TimeValuePair>> mapToFill,
+ final PathPatternTree scope)
throws MetadataException {
- return mtree.fillLastQueryMap(pattern, mapToFill);
+ return mtree.fillLastQueryMap(pattern, mapToFill, scope);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
index fd8d7c5..73bbce2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
@@ -1440,7 +1440,9 @@
@Override
public int fillLastQueryMap(
- final PartialPath pattern, final Map<PartialPath, Map<String, TimeValuePair>> mapToFill) {
+ final PartialPath pattern,
+ final Map<PartialPath, Map<String, TimeValuePair>> mapToFill,
+ final PathPatternTree scope) {
throw new UnsupportedOperationException("Not implemented");
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
index 63d1e4c..b6f1221 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
@@ -1085,12 +1085,13 @@
}
public int fillLastQueryMap(
- final PartialPath prefixPath, final Map<PartialPath, Map<String, TimeValuePair>> mapToFill)
+ final PartialPath prefixPath,
+ final Map<PartialPath, Map<String, TimeValuePair>> mapToFill,
+ final PathPatternTree scope)
throws MetadataException {
final int[] sensorNum = {0};
try (final EntityUpdater<IMemMNode> updater =
- new EntityUpdater<IMemMNode>(
- rootNode, prefixPath, store, true, SchemaConstant.ALL_MATCH_SCOPE) {
+ new EntityUpdater<IMemMNode>(rootNode, prefixPath, store, true, scope) {
@Override
protected void updateEntity(final IDeviceMNode<IMemMNode> node) {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
index 3b40a3b..53014d0 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
@@ -37,6 +37,7 @@
import java.util.List;
import java.util.Objects;
import java.util.Set;
+import java.util.stream.Collectors;
public class PathPatternTree {
@@ -94,7 +95,6 @@
appendBranchWithoutPrune(root, pathNodes, 0);
}
- /** Add a pathPattern (may contain wildcards) to pathPatternList. */
public void appendPathPattern(PartialPath pathPattern) {
if (useWildcard) {
boolean isExist = false;
@@ -120,7 +120,8 @@
for (PartialPath path : pathPatternList) {
appendBranchWithoutPrune(root, path.getNodes(), 0);
}
- pathPatternList.clear();
+ // Do not clear to avoid concurrent modification
+ pathPatternList = new LinkedList<>();
}
private void appendBranchWithoutPrune(
@@ -245,16 +246,24 @@
public List<PartialPath> getAllPathPatterns() {
List<PartialPath> result = new ArrayList<>();
Deque<String> ancestors = new ArrayDeque<>();
- searchPathPattern(root, ancestors, result);
+ searchPathPattern(root, ancestors, result, false);
return result;
}
- private void searchPathPattern(
+ public List<MeasurementPath> getAllPathPatterns(boolean asMeasurementPath) {
+ List<MeasurementPath> result = new ArrayList<>();
+ Deque<String> ancestors = new ArrayDeque<>();
+ searchPathPattern(root, ancestors, result, asMeasurementPath);
+ return result;
+ }
+
+ private <T extends PartialPath> void searchPathPattern(
PathPatternNode<Void, VoidSerializer> node,
Deque<String> ancestors,
- List<PartialPath> fullPaths) {
+ List<T> fullPaths,
+ boolean asMeasurementPath) {
if (node.isPathPattern()) {
- fullPaths.add(convertNodesToPartialPath(node, ancestors));
+ fullPaths.add((T) convertNodesToPartialPath(node, ancestors, asMeasurementPath));
if (node.isLeaf()) {
return;
}
@@ -262,23 +271,19 @@
ancestors.push(node.getName());
for (PathPatternNode<Void, VoidSerializer> child : node.getChildren().values()) {
- searchPathPattern(child, ancestors, fullPaths);
+ searchPathPattern(child, ancestors, fullPaths, asMeasurementPath);
}
ancestors.pop();
}
- public List<PartialPath> getOverlappedPathPatterns(PartialPath pattern) {
- if (pathPatternList.isEmpty()) {
- pathPatternList = getAllPathPatterns();
+ public List<PartialPath> getOverlappedPathPatterns(final PartialPath pattern) {
+ List<PartialPath> patternList = pathPatternList;
+ if (Objects.isNull(patternList) || patternList.isEmpty()) {
+ patternList = getAllPathPatterns();
+ pathPatternList = patternList;
}
- List<PartialPath> results = new ArrayList<>();
- for (PartialPath path : pathPatternList) {
- if (pattern.overlapWith(path)) {
- results.add(path);
- }
- }
- return results;
+ return patternList.stream().filter(pattern::overlapWith).collect(Collectors.toList());
}
private String convertNodesToString(List<String> nodes) {
@@ -290,14 +295,20 @@
}
private PartialPath convertNodesToPartialPath(
- PathPatternNode<Void, VoidSerializer> node, Deque<String> ancestors) {
+ PathPatternNode<Void, VoidSerializer> node,
+ Deque<String> ancestors,
+ boolean asMeasurementPath) {
Iterator<String> iterator = ancestors.descendingIterator();
List<String> nodeList = new ArrayList<>(ancestors.size() + 1);
while (iterator.hasNext()) {
nodeList.add(iterator.next());
}
nodeList.add(node.getName());
- return new PartialPath(nodeList.toArray(new String[0]));
+ if (asMeasurementPath) {
+ return new MeasurementPath(nodeList.toArray(new String[0]));
+ } else {
+ return new PartialPath(nodeList.toArray(new String[0]));
+ }
}
public boolean isOverlapWith(PathPatternTree patternTree) {
