add tag filter eq
diff --git a/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd b/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd index cb5815b..d8dcd64 100644 --- a/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd +++ b/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd
@@ -52,6 +52,11 @@ "dataType": "Binary", "javaBoxName": "String", "classSerializeName": "STRING" + }, + { + "dataType": "String", + "javaBoxName": "Tag", + "classSerializeName": "TAG" } ] } \ No newline at end of file
diff --git a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl index 77d394d..c4f1243 100644 --- a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl +++ b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
@@ -2,6 +2,12 @@ <#list filters as filter> <#assign className = "${filter.javaBoxName}FilterOperators"> <#assign filterName = "${filter.javaBoxName}Filter"> + <#if filter.javaBoxName == "Tag"> + <#assign javaClassName = "String"> + <#else> + <#assign javaClassName = "${filter.javaBoxName}"> + </#if> + <@pp.changeOutputFile name="/org/apache/tsfile/read/filter/operator/${className}.java" /> /* * Licensed to the Apache Software Foundation (ASF) under one @@ -71,7 +77,7 @@ protected ValueColumnCompareFilter(int measurementIndex, ${filter.dataType} constant) { super(measurementIndex); - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> this.constant = Objects.requireNonNull(constant, CONSTANT_CANNOT_BE_NULL_MSG); <#else> this.constant = constant; @@ -81,7 +87,7 @@ @SuppressWarnings("unchecked") protected ValueColumnCompareFilter(ByteBuffer buffer) { super(buffer); - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> this.constant = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer), CONSTANT_CANNOT_BE_NULL_MSG); <#else> this.constant = ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer); @@ -106,7 +112,7 @@ return false; } ValueColumnCompareFilter that = (ValueColumnCompareFilter) o; - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return Objects.equals(constant,that.constant); <#else> return constant == that.constant; @@ -142,7 +148,7 @@ @Override public boolean valueSatisfy(${filter.dataType} value) { - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return constant.equals(value); <#else> return constant == value; @@ -152,8 +158,8 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> - <#if filter.javaBoxName == "String"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> + <#if filter.javaBoxName == "String" || filter.javaBoxName == "Tag"> if(statistics.isEmpty()){ return false; } @@ -167,15 +173,15 @@ if(statistics.isEmpty()){ return false; } - return constant < (${filter.javaBoxName}) statistics.getMinValue() - || constant > (${filter.javaBoxName}) statistics.getMaxValue(); + return constant < (${javaClassName}) statistics.getMinValue() + || constant > (${javaClassName}) statistics.getMaxValue(); </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -190,8 +196,8 @@ if(statistics.isEmpty()){ return false; } - return constant == (${filter.javaBoxName}) statistics.getMinValue() - && constant == (${filter.javaBoxName}) statistics.getMaxValue(); + return constant == (${javaClassName}) statistics.getMinValue() + && constant == (${javaClassName}) statistics.getMaxValue(); </#if> } @@ -223,7 +229,7 @@ @Override public boolean valueSatisfy(${filter.dataType} value) { - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return !constant.equals(value); <#else> return constant != value; @@ -233,7 +239,7 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -249,15 +255,15 @@ return false; } // drop if this is a column where min = max = value - return constant == (${filter.javaBoxName}) statistics.getMinValue() - && constant == (${filter.javaBoxName}) statistics.getMaxValue(); + return constant == (${javaClassName}) statistics.getMinValue() + && constant == (${javaClassName}) statistics.getMaxValue(); </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -271,8 +277,8 @@ if(statistics.isEmpty()){ return false; } - return constant < (${filter.javaBoxName}) statistics.getMinValue() - || constant > (${filter.javaBoxName}) statistics.getMaxValue(); + return constant < (${javaClassName}) statistics.getMinValue() + || constant > (${javaClassName}) statistics.getMaxValue(); </#if> } @@ -307,7 +313,7 @@ public boolean valueSatisfy(${filter.dataType} value) { <#if filter.dataType == "boolean"> return Boolean.compare(constant,value) > 0; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> return constant.compareTo(value) > 0; <#else> return constant > value; @@ -317,7 +323,7 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -332,14 +338,14 @@ return false; } // drop if value <= min - return constant <= (${filter.javaBoxName}) statistics.getMinValue(); + return constant <= (${javaClassName}) statistics.getMinValue(); </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -352,7 +358,7 @@ if(statistics.isEmpty()){ return false; } - return constant > (${filter.javaBoxName}) statistics.getMaxValue(); + return constant > (${javaClassName}) statistics.getMaxValue(); </#if> } @@ -387,7 +393,7 @@ public boolean valueSatisfy(${filter.dataType} value) { <#if filter.dataType == "boolean"> return Boolean.compare(constant,value) >= 0; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> return constant.compareTo(value) >= 0; <#else> return constant >= value; @@ -397,7 +403,7 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -412,14 +418,14 @@ return false; } // drop if value < min - return constant < (${filter.javaBoxName}) statistics.getMinValue(); + return constant < (${javaClassName}) statistics.getMinValue(); </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -432,7 +438,7 @@ if(statistics.isEmpty()){ return false; } - return constant >= (${filter.javaBoxName}) statistics.getMaxValue(); + return constant >= (${javaClassName}) statistics.getMaxValue(); </#if> } @@ -467,7 +473,7 @@ public boolean valueSatisfy(${filter.dataType} value) { <#if filter.dataType == "boolean"> return Boolean.compare(constant,value) < 0; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> return constant.compareTo(value) < 0; <#else> return constant < value; @@ -477,7 +483,7 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -492,14 +498,14 @@ return false; } // drop if value >= max - return constant >= (${filter.javaBoxName}) statistics.getMaxValue(); + return constant >= (${javaClassName}) statistics.getMaxValue(); </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -512,7 +518,7 @@ if(statistics.isEmpty()){ return false; } - return constant < (${filter.javaBoxName}) statistics.getMinValue(); + return constant < (${javaClassName}) statistics.getMinValue(); </#if> } @@ -547,7 +553,7 @@ public boolean valueSatisfy(${filter.dataType} value) { <#if filter.dataType == "boolean"> return Boolean.compare(constant,value) <= 0; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> return constant.compareTo(value) <= 0; <#else> return constant <= value; @@ -557,7 +563,7 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -572,14 +578,14 @@ return false; } // drop if value > max - return constant > (${filter.javaBoxName}) statistics.getMaxValue(); + return constant > (${javaClassName}) statistics.getMaxValue(); </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -592,7 +598,7 @@ if(statistics.isEmpty()){ return false; } - return constant <= (${filter.javaBoxName}) statistics.getMinValue(); + return constant <= (${javaClassName}) statistics.getMinValue(); </#if> } @@ -615,7 +621,7 @@ protected ValueColumnRangeFilter(int measurementIndex, ${filter.dataType} min, ${filter.dataType} max) { super(measurementIndex); - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> this.min = Objects.requireNonNull(min,"min cannot be null"); this.max = Objects.requireNonNull(max,"max cannot be null"); <#else> @@ -627,7 +633,7 @@ @SuppressWarnings("unchecked") protected ValueColumnRangeFilter(ByteBuffer buffer) { super(buffer); - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> this.min = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer),"min cannot be null"); this.max = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer),"max cannot be null"); <#else> @@ -655,7 +661,7 @@ return false; } ValueColumnRangeFilter that = (ValueColumnRangeFilter) o; - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return min.equals(that.min) && max.equals(that.max); <#else> return min == that.min && max == that.max; @@ -696,7 +702,7 @@ <#if filter.dataType == "boolean"> return Boolean.compare(min,value) <= 0 && Boolean.compare(max,value) >= 0; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> return min.compareTo(value) <= 0 && max.compareTo(value) >= 0; <#else> @@ -707,7 +713,7 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -722,15 +728,15 @@ if(statistics.isEmpty()){ return false; } - return (${filter.javaBoxName}) statistics.getMaxValue() < min - || (${filter.javaBoxName}) statistics.getMinValue() > max; + return (${javaClassName}) statistics.getMaxValue() < min + || (${javaClassName}) statistics.getMinValue() > max; </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -744,8 +750,8 @@ if(statistics.isEmpty()){ return false; } - return (${filter.javaBoxName}) statistics.getMinValue() >= min - && (${filter.javaBoxName}) statistics.getMaxValue() <= max; + return (${javaClassName}) statistics.getMinValue() >= min + && (${javaClassName}) statistics.getMaxValue() <= max; </#if> } @@ -781,7 +787,7 @@ <#if filter.dataType == "boolean"> return Boolean.compare(min,value) > 0 || Boolean.compare(max,value) < 0; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> return min.compareTo(value) > 0 || max.compareTo(value) < 0; <#else> return min > value || max < value; @@ -791,7 +797,7 @@ @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -806,15 +812,15 @@ if(statistics.isEmpty()){ return false; } - return (${filter.javaBoxName}) statistics.getMinValue() >= min - && (${filter.javaBoxName}) statistics.getMaxValue() <= max; + return (${javaClassName}) statistics.getMinValue() >= min + && (${javaClassName}) statistics.getMaxValue() <= max; </#if> } @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - <#if filter.dataType == "boolean" || filter.dataType == "Binary"> + <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String"> <#if filter.javaBoxName == "String"> if(statistics.isEmpty()){ return false; @@ -828,8 +834,8 @@ if(statistics.isEmpty()){ return false; } - return (${filter.javaBoxName}) statistics.getMinValue() > max - || (${filter.javaBoxName}) statistics.getMaxValue() < min; + return (${javaClassName}) statistics.getMinValue() > max + || (${javaClassName}) statistics.getMaxValue() < min; </#if> } @@ -850,7 +856,7 @@ <#if filter.javaBoxName == "String"> protected final Set<${filter.dataType}> candidates; <#else> - protected final Set<${filter.javaBoxName}> candidates; + protected final Set<${javaClassName}> candidates; </#if> protected final ${filter.dataType} candidatesMin; @@ -859,7 +865,7 @@ <#if filter.javaBoxName == "String"> protected ValueColumnSetFilter(int measurementIndex, Set<${filter.dataType}> candidates) { <#else> - protected ValueColumnSetFilter(int measurementIndex, Set<${filter.javaBoxName}> candidates) { + protected ValueColumnSetFilter(int measurementIndex, Set<${javaClassName}> candidates) { </#if> super(measurementIndex); this.candidates = candidates; @@ -867,18 +873,18 @@ <#if filter.javaBoxName == "String"> Set<${filter.dataType}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet()); <#else> - Set<${filter.javaBoxName}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet()); + Set<${javaClassName}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet()); </#if> <#if filter.dataType == "boolean"> // BooleanStatistics is not available this.candidatesMin = false; this.candidatesMax = false; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : null; this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : null; <#else> - this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : ${filter.javaBoxName}.MIN_VALUE; - this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : ${filter.javaBoxName}.MAX_VALUE; + this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : ${javaClassName}.MIN_VALUE; + this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : ${javaClassName}.MAX_VALUE; </#if> } @@ -888,18 +894,18 @@ <#if filter.javaBoxName == "String"> this.candidates = ReadWriteIOUtils.read${filter.dataType}Set(buffer); <#else> - this.candidates = ReadWriteIOUtils.read${filter.javaBoxName}Set(buffer); + this.candidates = ReadWriteIOUtils.read${javaClassName}Set(buffer); </#if> <#if filter.dataType == "boolean"> // BooleanStatistics is not available this.candidatesMin = false; this.candidatesMax = false; - <#elseif filter.dataType == "Binary"> + <#elseif filter.dataType == "Binary" || filter.dataType == "String"> this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : null; this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : null; <#else> - this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : ${filter.javaBoxName}.MAX_VALUE; - this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : ${filter.javaBoxName}.MAX_VALUE; + this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : ${javaClassName}.MAX_VALUE; + this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : ${javaClassName}.MAX_VALUE; </#if> if(hasNull){ this.candidates.add(null); @@ -913,7 +919,7 @@ <#if filter.javaBoxName == "String"> ReadWriteIOUtils.write${filter.dataType}Set(candidates, outputStream); <#else> - ReadWriteIOUtils.write${filter.javaBoxName}Set(candidates, outputStream); + ReadWriteIOUtils.write${javaClassName}Set(candidates, outputStream); </#if> } @@ -951,7 +957,7 @@ super(measurementIndex, candidates); } <#else> - public ValueIn(int measurementIndex, Set<${filter.javaBoxName}> candidates) { + public ValueIn(int measurementIndex, Set<${javaClassName}> candidates) { super(measurementIndex, candidates); } </#if> @@ -962,7 +968,7 @@ @Override public boolean valueSatisfy(Object value){ - return candidates.contains((${filter.javaBoxName}) value); + return candidates.contains((${javaClassName}) value); } @Override @@ -990,12 +996,12 @@ if (statistics.isPresent()) { Statistics<? extends Serializable> stat = statistics.get(); - <#if filter.dataType == "Binary" && filter.javaBoxName == "String"> + <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")> ${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue(); ${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue(); <#else> - ${filter.javaBoxName} valuesMin = (${filter.javaBoxName}) stat.getMinValue(); - ${filter.javaBoxName} valuesMax = (${filter.javaBoxName}) stat.getMaxValue(); + ${javaClassName} valuesMin = (${javaClassName}) stat.getMinValue(); + ${javaClassName} valuesMax = (${javaClassName}) stat.getMaxValue(); </#if> // All values are same if (valuesMin.equals(valuesMax)) { @@ -1003,7 +1009,7 @@ } else { if (!candidates.isEmpty()) { // All values are less than min, or greater than max - <#if filter.dataType == "Binary" && filter.javaBoxName == "String"> + <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")> return candidatesMin.compareTo(valuesMax) > 0 || candidatesMax.compareTo(valuesMin) < 0; <#else> @@ -1044,12 +1050,12 @@ // All values are same if (statistics.isPresent()) { Statistics<? extends Serializable> stat = statistics.get(); - <#if filter.dataType == "Binary" && filter.javaBoxName == "String"> + <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")> ${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue(); ${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue(); <#else> - ${filter.javaBoxName} valuesMin = (${filter.javaBoxName}) stat.getMinValue(); - ${filter.javaBoxName} valuesMax = (${filter.javaBoxName}) stat.getMaxValue(); + ${javaClassName} valuesMin = (${javaClassName}) stat.getMinValue(); + ${javaClassName} valuesMax = (${javaClassName}) stat.getMaxValue(); </#if> // All values are same if (valuesMin.equals(valuesMax)) { @@ -1095,7 +1101,7 @@ super(measurementIndex, candidates); } <#else> - public ValueNotIn(int measurementIndex, Set<${filter.javaBoxName}> candidates) { + public ValueNotIn(int measurementIndex, Set<${javaClassName}> candidates) { super(measurementIndex, candidates); } </#if> @@ -1106,7 +1112,7 @@ @Override public boolean valueSatisfy(Object value){ - return !candidates.contains((${filter.javaBoxName}) value); + return !candidates.contains((${javaClassName}) value); } @Override @@ -1203,7 +1209,7 @@ @Override public boolean valueSatisfy(${filter.dataType} value) { - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find(); <#else> return pattern.matcher(new MatcherInput(String.valueOf(value), new AccessCount())).find(); @@ -1248,7 +1254,7 @@ @Override public boolean valueSatisfy(${filter.dataType} value) { - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return !pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find(); <#else> return !pattern.matcher(new MatcherInput(String.valueOf(value), new AccessCount())).find(); @@ -1353,7 +1359,7 @@ @Override public boolean valueSatisfy(${filter.dataType} value) { - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return pattern.getMatcher().match(value.toString().getBytes()); <#else> return pattern.getMatcher().match(String.valueOf(value).getBytes()); @@ -1398,7 +1404,7 @@ @Override public boolean valueSatisfy(${filter.dataType} value) { - <#if filter.dataType == "Binary"> + <#if filter.dataType == "Binary" || filter.dataType == "String"> return !pattern.getMatcher().match(value.toString().getBytes()); <#else> return !pattern.getMatcher().match(String.valueOf(value).getBytes());
diff --git a/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl index e1fd9fe..d057484 100644 --- a/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl +++ b/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl
@@ -66,6 +66,7 @@ protected abstract boolean valueSatisfy(${filter.dataType} value); + <#if filter.javaBoxName != "Tag"> @Override public boolean[] satisfyTsBlock(boolean[] selection, TsBlock tsBlock) { Column valueColumn = tsBlock.getValueColumns()[measurementIndex]; @@ -83,6 +84,12 @@ } return satisfyInfo; } + <#else > + @Override + public boolean[] satisfyTsBlock(boolean[] selection, TsBlock tsBlock) { + throw new IllegalArgumentException("TagFilter cannot be applied to TsBlock"); + } + </#if> @Override public void serialize(DataOutputStream outputStream) throws IOException {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java index 4445bf9..74c99ef 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
@@ -24,7 +24,7 @@ import org.apache.tsfile.file.metadata.MetadataIndexNode; import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType; import org.apache.tsfile.read.TsFileSequenceReader; -import org.apache.tsfile.read.expression.ExpressionTree; +import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -43,15 +43,15 @@ private final TsFileSequenceReader tsFileSequenceReader; private final Queue<MetadataIndexNode> metadataIndexNodes = new ArrayDeque<>(); private final Queue<Pair<IDeviceID, MetadataIndexNode>> resultCache = new ArrayDeque<>(); - private final ExpressionTree idFilter; + private final Filter tagFilter; public DeviceMetaIterator( TsFileSequenceReader tsFileSequenceReader, MetadataIndexNode metadataIndexNode, - ExpressionTree idFilter) { + Filter tagFilter) { this.tsFileSequenceReader = tsFileSequenceReader; this.metadataIndexNodes.add(metadataIndexNode); - this.idFilter = idFilter; + this.tagFilter = tagFilter; } @Override @@ -74,7 +74,8 @@ for (int i = 0; i < leafChildren.size(); i++) { IMetadataIndexEntry child = leafChildren.get(i); final IDeviceID deviceID = ((DeviceMetadataIndexEntry) child).getDeviceID(); - if (idFilter != null && !idFilter.satisfy(deviceID)) { + // time is only a placeholder here + if (tagFilter != null && !tagFilter.satisfyRow(0, deviceID.getSegments())) { continue; } @@ -114,6 +115,7 @@ if (!resultCache.isEmpty()) { return; } + break; case INTERNAL_DEVICE: loadInternalNode(currentNode); break;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java index 3503e49..3f74178 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
@@ -28,7 +28,7 @@ import org.apache.tsfile.file.metadata.TsFileMetadata; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.TimeRange; -import org.apache.tsfile.read.expression.ExpressionTree; +import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; import java.io.IOException; @@ -87,5 +87,5 @@ void clear(); Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator( - MetadataIndexNode root, ExpressionTree idFilter); + MetadataIndexNode root, Filter idFilter); }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java index ae7fb9e..68e1953 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -34,7 +34,7 @@ import org.apache.tsfile.read.TsFileSequenceReader.LocateStatus; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.TimeRange; -import org.apache.tsfile.read.expression.ExpressionTree; +import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.utils.Pair; import java.io.IOException; @@ -306,7 +306,7 @@ @Override public Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator( - MetadataIndexNode root, ExpressionTree idFilter) { - return new DeviceMetaIterator(tsFileReader, root, idFilter); + MetadataIndexNode root, Filter tagFilter) { + return new DeviceMetaIterator(tsFileReader, root, tagFilter); } }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java index cc48210..a5aac76 100755 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java
@@ -77,6 +77,8 @@ public abstract boolean satisfyBinary(long time, Binary value); + public abstract boolean satisfyString(long time, String value); + /** * To examine whether the row(with time and values) is satisfied with the filter. *
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java index 18d444d..a23ee31 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java
@@ -75,6 +75,11 @@ } @Override + public boolean satisfyString(long time, String value) { + return timeSatisfy(time); + } + + @Override public boolean satisfyRow(long time, Object[] values) { // only use time to filter return timeSatisfy(time);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java index e1ae98b..d715352 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java
@@ -85,13 +85,18 @@ } @Override + public boolean satisfyString(long time, String value) { + throw new UnSupportedDataTypeException(getClass().getName()); + } + + @Override public boolean satisfyBinary(long time, Binary value) { throw new UnSupportedDataTypeException(getClass().getName()); } @Override public boolean satisfyRow(long time, Object[] values) { - return satisfy(time, values[measurementIndex]); + return satisfy(time, measurementIndex < values.length ? values[measurementIndex] : null); } @Override
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java new file mode 100644 index 0000000..a315950 --- /dev/null +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java
@@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.read.filter.factory; + +import org.apache.tsfile.annotations.TsFileApi; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.filter.operator.TagFilterOperators.ValueEq; +import org.apache.tsfile.write.schema.IMeasurementSchema; + +public class TagFilterBuilder { + private TableSchema tableSchema; + + public TagFilterBuilder(TableSchema tableSchema) { + this.tableSchema = tableSchema; + } + + @TsFileApi + public Filter eq(String columnName, Object value) { + int idColumnOrder = tableSchema.findIdColumnOrder(columnName); + if (idColumnOrder == -1) { + throw new IllegalArgumentException("Column '" + columnName + "' is not a tag column"); + } + IMeasurementSchema columnSchema = tableSchema.findColumnSchema(columnName); + + // +1 for table name + return new ValueEq(idColumnOrder + 1, (String) value); + } +}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java index de0f338..8df4e50 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java
@@ -78,6 +78,11 @@ } @Override + public boolean satisfyString(long time, String value) { + return left.satisfyString(time, value) && right.satisfyString(time, value); + } + + @Override public boolean satisfyRow(long time, Object[] values) { return left.satisfyRow(time, values) && right.satisfyRow(time, values); }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java index 0091891..e02a369 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java
@@ -91,6 +91,11 @@ } @Override + public boolean satisfyString(long time, String value) { + return !filter.satisfyString(time, value); + } + + @Override public boolean satisfyBooleanRow(long time, boolean[] values) { return !filter.satisfyBooleanRow(time, values); }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java index f0f1c2f..97a7875 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java
@@ -78,6 +78,11 @@ } @Override + public boolean satisfyString(long time, String value) { + return left.satisfyString(time, value) || right.satisfyString(time, value); + } + + @Override public boolean satisfyRow(long time, Object[] values) { return left.satisfyRow(time, values) || right.satisfyRow(time, values); }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java index 0f2d72b..2e5d070 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java
@@ -20,9 +20,11 @@ package org.apache.tsfile.read.query.dataset; import org.apache.tsfile.annotations.TsFileApi; +import org.apache.tsfile.write.record.TSRecord; import java.io.IOException; import java.time.LocalDate; +import java.util.Iterator; public interface ResultSet extends AutoCloseable { @@ -87,5 +89,8 @@ boolean isNull(int columnIndex); @TsFileApi - public abstract void close(); + void close(); + + @TsFileApi + Iterator<TSRecord> recordIterator(); }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java index 5e05077..8a585df 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java
@@ -26,27 +26,36 @@ import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.read.reader.block.TsBlockReader; +import org.apache.tsfile.write.record.TSRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Iterator; import java.util.List; +import java.util.NoSuchElementException; public class TableResultSet extends AbstractResultSet { + private static final Logger LOG = LoggerFactory.getLogger(TableResultSet.class); private TsBlockReader tsBlockReader; private IPointReader tsBlockPointReader; private List<String> columnNameList; private List<TSDataType> dataTypeList; + private String tableName; public TableResultSet( - TsBlockReader tsBlockReader, List<String> columnNameList, List<TSDataType> dataTypeList) { + TsBlockReader tsBlockReader, + List<String> columnNameList, + List<TSDataType> dataTypeList, + String tableName) { super(columnNameList, dataTypeList); this.tsBlockReader = tsBlockReader; this.columnNameList = columnNameList; this.dataTypeList = dataTypeList; + this.tableName = tableName; } @Override @@ -85,4 +94,89 @@ LOG.error("Failed to close tsBlockReader"); } } + + @Override + public Iterator<TSRecord> recordIterator() { + return new RecordIterator(); + } + + private class RecordIterator implements Iterator<TSRecord> { + + private TSRecord cachedRecord = null; + private boolean exhausted = false; + + @Override + public boolean hasNext() { + if (cachedRecord != null) { + return true; + } + if (exhausted) { + return false; + } + + try { + return cacheNextRecord(); + } catch (IOException e) { + throw new NoSuchElementException(e.toString()); + } + } + + private boolean cacheNextRecord() throws IOException { + boolean next = TableResultSet.this.next(); + if (!next) { + exhausted = true; + return false; + } + + cachedRecord = new TSRecord(tableName, getLong("Time")); + for (int i = 0; i < columnNameList.size(); i++) { + Field field = currentRow.getFields().get(i); + if (field != null) { + switch (field.getDataType()) { + case INT32: + case DATE: + cachedRecord.addPoint(columnNameList.get(i), field.getIntV()); + break; + case INT64: + case TIMESTAMP: + cachedRecord.addPoint(columnNameList.get(i), field.getLongV()); + break; + case FLOAT: + cachedRecord.addPoint(columnNameList.get(i), field.getFloatV()); + break; + case DOUBLE: + cachedRecord.addPoint(columnNameList.get(i), field.getDoubleV()); + break; + case STRING: + case TEXT: + cachedRecord.addPoint(columnNameList.get(i), field.getStringValue()); + break; + case BLOB: + cachedRecord.addPoint(columnNameList.get(i), field.getBinaryV().getValues()); + break; + case BOOLEAN: + cachedRecord.addPoint(columnNameList.get(i), field.getBoolV()); + break; + case VECTOR: + case UNKNOWN: + default: + break; + } + } else { + cachedRecord.dataPointList.add(null); + } + } + return true; + } + + @Override + public TSRecord next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + TSRecord ret = cachedRecord; + cachedRecord = null; + return ret; + } + } }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java index 0f3aade..1f12cc6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java
@@ -21,8 +21,10 @@ import org.apache.tsfile.annotations.TsFileApi; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.write.record.TSRecord; import java.io.IOException; +import java.util.Iterator; import java.util.stream.Collectors; public class TreeResultSet extends AbstractResultSet { @@ -51,4 +53,9 @@ public void close() { // nothing to be done } + + @Override + public Iterator<TSRecord> recordIterator() { + return null; + } }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java index a192a36..c50d1bf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
@@ -29,6 +29,7 @@ import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.controller.IMetadataQuerier; import org.apache.tsfile.read.expression.ExpressionTree; +import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.query.executor.task.DeviceTaskIterator; import org.apache.tsfile.read.reader.block.DeviceOrderedTsBlockReader; import org.apache.tsfile.read.reader.block.TsBlockReader; @@ -62,7 +63,7 @@ * @param tableName table to query * @param columns columns to query (ID or MEASUREMENT) * @param timeFilter time predicate - * @param idFilter id predicate + * @param tagFilter id predicate * @param measurementFilter measurement predicate * @return an iterator of TsBlocks * @throws ReadProcessException if the read process fails @@ -71,7 +72,7 @@ String tableName, List<String> columns, ExpressionTree timeFilter, - ExpressionTree idFilter, + Filter tagFilter, ExpressionTree measurementFilter) throws ReadProcessException { TsFileMetadata fileMetadata = metadataQuerier.getWholeFileMetadata(); @@ -90,7 +91,7 @@ DeviceTaskIterator deviceTaskIterator = new DeviceTaskIterator( - columns, tableRoot, columnMapping, metadataQuerier, idFilter, tableSchema); + columns, tableRoot, columnMapping, metadataQuerier, tagFilter, tableSchema); switch (tableQueryOrdering) { case DEVICE: return new DeviceOrderedTsBlockReader(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java index 8ee4465..399cd32 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
@@ -23,7 +23,7 @@ import org.apache.tsfile.file.metadata.MetadataIndexNode; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.controller.IMetadataQuerier; -import org.apache.tsfile.read.expression.ExpressionTree; +import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.query.executor.TableQueryExecutor.ColumnMapping; import org.apache.tsfile.utils.Pair; @@ -41,11 +41,11 @@ MetadataIndexNode indexRoot, ColumnMapping columnMapping, IMetadataQuerier metadataQuerier, - ExpressionTree idFilter, + Filter tagFilter, TableSchema tableSchema) { this.columnNames = columnNames; this.columnMapping = columnMapping; - this.deviceMetaIterator = metadataQuerier.deviceIterator(indexRoot, idFilter); + this.deviceMetaIterator = metadataQuerier.deviceIterator(indexRoot, tagFilter); this.tableSchema = tableSchema; }
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java index 894353f..927ac89 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
@@ -31,6 +31,7 @@ import org.apache.tsfile.read.controller.IMetadataQuerier; import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl; import org.apache.tsfile.read.expression.ExpressionTree; +import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.read.query.dataset.TableResultSet; import org.apache.tsfile.read.query.executor.TableQueryExecutor; @@ -79,6 +80,13 @@ @TsFileApi public ResultSet query(String tableName, List<String> columnNames, long startTime, long endTime) throws IOException, NoTableException, NoMeasurementException, ReadProcessException { + return query(tableName, columnNames, startTime, endTime, null); + } + + @Override + public ResultSet query( + String tableName, List<String> columnNames, long startTime, long endTime, Filter tagFilter) + throws ReadProcessException, IOException, NoTableException, NoMeasurementException { String lowerCaseTableName = tableName.toLowerCase(); TableSchema tableSchema = fileReader.getTableSchemaMap().get(lowerCaseTableName); if (tableSchema == null) { @@ -100,9 +108,9 @@ lowerCaseTableName, lowerCaseColumnNames, new ExpressionTree.TimeBetweenAnd(startTime, endTime), - null, + tagFilter, null); - return new TableResultSet(tsBlockReader, columnNames, dataTypeList); + return new TableResultSet(tsBlockReader, columnNames, dataTypeList, tableName); } @Override
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java index 885e845..995f73f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
@@ -24,6 +24,7 @@ import org.apache.tsfile.exception.write.NoMeasurementException; import org.apache.tsfile.exception.write.NoTableException; import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.query.dataset.ResultSet; import java.io.IOException; @@ -37,6 +38,11 @@ throws ReadProcessException, IOException, NoTableException, NoMeasurementException; @TsFileApi + ResultSet query( + String tableName, List<String> columnNames, long startTime, long endTime, Filter tagFilter) + throws ReadProcessException, IOException, NoTableException, NoMeasurementException; + + @TsFileApi Optional<TableSchema> getTableSchemas(String tableName) throws IOException; @TsFileApi
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java index 6e38e51..e6b72e4 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -997,6 +997,18 @@ return set; } + public static Set<String> readStringSet(ByteBuffer buffer) { + int size = readInt(buffer); + if (size <= 0) { + return Collections.emptySet(); + } + Set<String> set = new HashSet<>(); + for (int i = 0; i < size; i++) { + set.add(readString(buffer)); + } + return set; + } + // read object set with self define length public static <T> Set<T> readObjectSet(ByteBuffer buffer) { int size = readInt(buffer); @@ -1082,6 +1094,16 @@ } } + public static void writeStringSet(Set<String> set, DataOutputStream outputStream) + throws IOException { + write(set.contains(null) ? set.size() - 1 : set.size(), outputStream); + for (String e : set) { + if (e != null) { + write(e, outputStream); + } + } + } + public static CompressionType readCompressionType(InputStream inputStream) throws IOException { byte n = readByte(inputStream); return CompressionType.deserialize(n); @@ -1145,6 +1167,7 @@ BINARY, BOOLEAN, STRING, + TAG, NULL }
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java index a9f8766..090d7f8 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
@@ -22,10 +22,12 @@ import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.filter.factory.TagFilterBuilder; import org.apache.tsfile.read.query.dataset.ResultSet; import org.apache.tsfile.read.query.dataset.ResultSetMetadata; import org.apache.tsfile.read.v4.DeviceTableModelReader; import org.apache.tsfile.utils.TsFileGeneratorForTest; +import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.v4.ITsFileWriter; @@ -40,6 +42,7 @@ import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; +import java.util.Iterator; public class ResultSetTest { @@ -140,6 +143,135 @@ } @Test + public void testQueryTableWithTagFilter() throws Exception { + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("id1", TSDataType.STRING), + new MeasurementSchema("id2", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.BOOLEAN)), + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = + new Tablet( + Arrays.asList("id1", "id2", "s1", "s2"), + Arrays.asList( + TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN), + 1024); + tablet.addTimestamp(0, 0); + tablet.addValue("id1", 0, "id_field1"); + tablet.addValue("id2", 0, "id_field2"); + tablet.addValue("s1", 0, true); + tablet.addValue("s2", 0, false); + + tablet.addTimestamp(1, 1); + tablet.addValue("id1", 1, "id_field1_2"); + tablet.addValue("s2", 1, true); + + tablet.addTimestamp(2, 2); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) { + writer.write(tablet); + } + + // eq + TagFilterBuilder filterBuilder = new TagFilterBuilder(tableSchema); + try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile); + ResultSet resultSet = + tsFileReader.query( + "T1", + Arrays.asList("ID1", "ID2", "S2", "S1"), + 0, + 2, + filterBuilder.eq("ID1", "id_field1"))) { + + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(0, resultSet.getLong(1)); + Assert.assertEquals("id_field1", resultSet.getString(2)); + Assert.assertEquals("id_field2", resultSet.getString(3)); + Assert.assertFalse(resultSet.getBoolean(4)); + Assert.assertTrue(resultSet.getBoolean(5)); + } + } + + @Test + public void testQueryTableByIterator() throws Exception { + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("id1", TSDataType.STRING), + new MeasurementSchema("id2", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.BOOLEAN)), + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = + new Tablet( + Arrays.asList("id1", "id2", "s1", "s2"), + Arrays.asList( + TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN), + 1024); + tablet.addTimestamp(0, 0); + tablet.addValue("id1", 0, "id_field1"); + tablet.addValue("id2", 0, "id_field2"); + tablet.addValue("s1", 0, true); + tablet.addValue("s2", 0, false); + + tablet.addTimestamp(1, 1); + tablet.addValue("id1", 1, "id_field1_2"); + tablet.addValue("s2", 1, true); + + tablet.addTimestamp(2, 2); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) { + writer.write(tablet); + } + + try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile); + ResultSet resultSet = + tsFileReader.query("T1", Arrays.asList("ID1", "ID2", "S2", "S1"), 0, 2); ) { + Iterator<TSRecord> tsRecordIterator = resultSet.recordIterator(); + + Assert.assertTrue(tsRecordIterator.hasNext()); + TSRecord tsRecord = tsRecordIterator.next(); + Assert.assertEquals(2, tsRecord.time); + Assert.assertNull(tsRecord.dataPointList.get(0)); + Assert.assertNull(tsRecord.dataPointList.get(1)); + Assert.assertNull(tsRecord.dataPointList.get(2)); + Assert.assertNull(tsRecord.dataPointList.get(3)); + + Assert.assertTrue(tsRecordIterator.hasNext()); + tsRecord = tsRecordIterator.next(); + Assert.assertEquals(0, tsRecord.time); + Assert.assertEquals("id_field1", tsRecord.dataPointList.get(0).getValue().toString()); + Assert.assertEquals("id_field2", tsRecord.dataPointList.get(1).getValue().toString()); + Assert.assertFalse((Boolean) tsRecord.dataPointList.get(2).getValue()); + Assert.assertTrue((Boolean) tsRecord.dataPointList.get(3).getValue()); + + Assert.assertTrue(tsRecordIterator.hasNext()); + tsRecord = tsRecordIterator.next(); + Assert.assertEquals(1, tsRecord.time); + Assert.assertEquals("id_field1_2", tsRecord.dataPointList.get(0).getValue().toString()); + Assert.assertNull(tsRecord.dataPointList.get(1)); + Assert.assertTrue((Boolean) tsRecord.dataPointList.get(2).getValue()); + Assert.assertNull(tsRecord.dataPointList.get(3)); + + Assert.assertFalse(tsRecordIterator.hasNext()); + } + } + + @Test public void testQueryWithMaxValue() throws Exception { TableSchema tableSchema = new TableSchema(