Pipe: allow exclusion-only pattern lists (#17154)
* fix(pipe): allow exclusion-only pattern lists
* improve IT
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java
index 9079dae..00e3078 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java
@@ -385,4 +385,60 @@
"Time,root.db.d1.t,root.db.d2.s,",
expectedResSet);
}
+
+ @Test
+ public void testIoTDBPatternWithExclusionOnlyHistoricalData() throws Exception {
+ final Map<String, String> sourceAttributes = new HashMap<>();
+ sourceAttributes.put("source.pattern.exclusion", "root.db.d1.s*, root.db.d3.*");
+ sourceAttributes.put("source.inclusion", "data.insert");
+ sourceAttributes.put("user", "root");
+
+ final List<String> insertQueries =
+ Arrays.asList(
+ "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)",
+ "insert into root.db.d2(time, s) values (2, 2)",
+ "insert into root.db.d3(time, s) values (3, 3)",
+ "insert into root.db1.d1(time, s) values (4, 4)");
+
+ final Set<String> expectedResSet = new HashSet<>();
+ expectedResSet.add("1,null,1.0,null,");
+ expectedResSet.add("2,null,null,2.0,");
+ expectedResSet.add("4,4.0,null,null,");
+
+ testPipeWithMultiplePatterns(
+ sourceAttributes,
+ insertQueries,
+ true, // isHistorical = true
+ "select * from root.db1.**,root.db.**",
+ "Time,root.db1.d1.s,root.db.d1.t,root.db.d2.s,",
+ expectedResSet);
+ }
+
+ @Test
+ public void testIoTDBPatternWithExclusionOnlyRealtimeData() throws Exception {
+ final Map<String, String> sourceAttributes = new HashMap<>();
+ sourceAttributes.put("source.pattern.exclusion", "root.db.d1.s*, root.db.d3.*");
+ sourceAttributes.put("source.inclusion", "data.insert");
+ sourceAttributes.put("user", "root");
+
+ final List<String> insertQueries =
+ Arrays.asList(
+ "insert into root.db.d1(time, s, s1, t) values (1, 1, 1, 1)",
+ "insert into root.db.d2(time, s) values (2, 2)",
+ "insert into root.db.d3(time, s) values (3, 3)",
+ "insert into root.db1.d1(time, s) values (4, 4)");
+
+ final Set<String> expectedResSet = new HashSet<>();
+ expectedResSet.add("1,null,1.0,null,");
+ expectedResSet.add("2,null,null,2.0,");
+ expectedResSet.add("4,4.0,null,null,");
+
+ testPipeWithMultiplePatterns(
+ sourceAttributes,
+ insertQueries,
+ false, // isHistorical = false
+ "select * from root.db1.**,root.db.**",
+ "Time,root.db1.d1.s,root.db.d1.t,root.db.d2.s,",
+ expectedResSet);
+ }
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java
index 0cea3b3..5d6c808 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java
@@ -156,10 +156,16 @@
final boolean hasPatternInclusionKey =
sourceParameters.hasAnyAttributes(
EXTRACTOR_PATTERN_INCLUSION_KEY, SOURCE_PATTERN_INCLUSION_KEY);
+ final boolean hasPatternExclusionKey =
+ sourceParameters.hasAnyAttributes(
+ EXTRACTOR_PATTERN_EXCLUSION_KEY, SOURCE_PATTERN_EXCLUSION_KEY);
final boolean hasLegacyPathKey =
sourceParameters.hasAnyAttributes(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY);
final boolean hasLegacyPatternKey =
sourceParameters.hasAnyAttributes(EXTRACTOR_PATTERN_KEY, SOURCE_PATTERN_KEY);
+ final boolean usePatternSyntax =
+ hasPatternInclusionKey
+ || (hasPatternExclusionKey && !hasLegacyPathKey && !hasLegacyPatternKey);
if (hasPatternInclusionKey && (hasLegacyPathKey || hasLegacyPatternKey)) {
final String msg =
@@ -172,7 +178,7 @@
// 1. Parse INCLUSION patterns into a list
List<TreePattern> inclusionPatterns =
- hasPatternInclusionKey
+ usePatternSyntax
? parseIoTDBPatternList(
sourceParameters.getStringByKeys(
EXTRACTOR_PATTERN_INCLUSION_KEY, SOURCE_PATTERN_INCLUSION_KEY),
@@ -198,7 +204,7 @@
}
// 2. Parse EXCLUSION patterns into a list
- if (hasPatternInclusionKey
+ if (usePatternSyntax
&& sourceParameters.hasAnyAttributes(
EXTRACTOR_PATH_EXCLUSION_KEY, SOURCE_PATH_EXCLUSION_KEY)) {
final String msg =
@@ -210,7 +216,7 @@
}
List<TreePattern> exclusionPatterns =
- hasPatternInclusionKey
+ usePatternSyntax
? parseIoTDBPatternList(
sourceParameters.getStringByKeys(
EXTRACTOR_PATTERN_EXCLUSION_KEY, SOURCE_PATTERN_EXCLUSION_KEY),
