add tag filter eq
diff --git a/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd b/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd
index cb5815b..d8dcd64 100644
--- a/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd
+++ b/java/tsfile/src/main/codegen/dataModel/AllFilter.tdd
@@ -52,6 +52,11 @@
"dataType": "Binary",
"javaBoxName": "String",
"classSerializeName": "STRING"
+ },
+ {
+ "dataType": "String",
+ "javaBoxName": "Tag",
+ "classSerializeName": "TAG"
}
]
}
\ No newline at end of file
diff --git a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
index 77d394d..c4f1243 100644
--- a/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
+++ b/java/tsfile/src/main/codegen/templates/FilterOperatorsTemplate.ftl
@@ -2,6 +2,12 @@
<#list filters as filter>
<#assign className = "${filter.javaBoxName}FilterOperators">
<#assign filterName = "${filter.javaBoxName}Filter">
+ <#if filter.javaBoxName == "Tag">
+ <#assign javaClassName = "String">
+ <#else>
+ <#assign javaClassName = "${filter.javaBoxName}">
+ </#if>
+
<@pp.changeOutputFile name="/org/apache/tsfile/read/filter/operator/${className}.java" />
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -71,7 +77,7 @@
protected ValueColumnCompareFilter(int measurementIndex, ${filter.dataType} constant) {
super(measurementIndex);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.constant = Objects.requireNonNull(constant, CONSTANT_CANNOT_BE_NULL_MSG);
<#else>
this.constant = constant;
@@ -81,7 +87,7 @@
@SuppressWarnings("unchecked")
protected ValueColumnCompareFilter(ByteBuffer buffer) {
super(buffer);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.constant = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer), CONSTANT_CANNOT_BE_NULL_MSG);
<#else>
this.constant = ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer);
@@ -106,7 +112,7 @@
return false;
}
ValueColumnCompareFilter that = (ValueColumnCompareFilter) o;
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return Objects.equals(constant,that.constant);
<#else>
return constant == that.constant;
@@ -142,7 +148,7 @@
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return constant.equals(value);
<#else>
return constant == value;
@@ -152,8 +158,8 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
- <#if filter.javaBoxName == "String">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
+ <#if filter.javaBoxName == "String" || filter.javaBoxName == "Tag">
if(statistics.isEmpty()){
return false;
}
@@ -167,15 +173,15 @@
if(statistics.isEmpty()){
return false;
}
- return constant < (${filter.javaBoxName}) statistics.getMinValue()
- || constant > (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant < (${javaClassName}) statistics.getMinValue()
+ || constant > (${javaClassName}) statistics.getMaxValue();
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -190,8 +196,8 @@
if(statistics.isEmpty()){
return false;
}
- return constant == (${filter.javaBoxName}) statistics.getMinValue()
- && constant == (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant == (${javaClassName}) statistics.getMinValue()
+ && constant == (${javaClassName}) statistics.getMaxValue();
</#if>
}
@@ -223,7 +229,7 @@
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return !constant.equals(value);
<#else>
return constant != value;
@@ -233,7 +239,7 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -249,15 +255,15 @@
return false;
}
// drop if this is a column where min = max = value
- return constant == (${filter.javaBoxName}) statistics.getMinValue()
- && constant == (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant == (${javaClassName}) statistics.getMinValue()
+ && constant == (${javaClassName}) statistics.getMaxValue();
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -271,8 +277,8 @@
if(statistics.isEmpty()){
return false;
}
- return constant < (${filter.javaBoxName}) statistics.getMinValue()
- || constant > (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant < (${javaClassName}) statistics.getMinValue()
+ || constant > (${javaClassName}) statistics.getMaxValue();
</#if>
}
@@ -307,7 +313,7 @@
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) > 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) > 0;
<#else>
return constant > value;
@@ -317,7 +323,7 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -332,14 +338,14 @@
return false;
}
// drop if value <= min
- return constant <= (${filter.javaBoxName}) statistics.getMinValue();
+ return constant <= (${javaClassName}) statistics.getMinValue();
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -352,7 +358,7 @@
if(statistics.isEmpty()){
return false;
}
- return constant > (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant > (${javaClassName}) statistics.getMaxValue();
</#if>
}
@@ -387,7 +393,7 @@
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) >= 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) >= 0;
<#else>
return constant >= value;
@@ -397,7 +403,7 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -412,14 +418,14 @@
return false;
}
// drop if value < min
- return constant < (${filter.javaBoxName}) statistics.getMinValue();
+ return constant < (${javaClassName}) statistics.getMinValue();
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -432,7 +438,7 @@
if(statistics.isEmpty()){
return false;
}
- return constant >= (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant >= (${javaClassName}) statistics.getMaxValue();
</#if>
}
@@ -467,7 +473,7 @@
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) < 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) < 0;
<#else>
return constant < value;
@@ -477,7 +483,7 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -492,14 +498,14 @@
return false;
}
// drop if value >= max
- return constant >= (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant >= (${javaClassName}) statistics.getMaxValue();
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -512,7 +518,7 @@
if(statistics.isEmpty()){
return false;
}
- return constant < (${filter.javaBoxName}) statistics.getMinValue();
+ return constant < (${javaClassName}) statistics.getMinValue();
</#if>
}
@@ -547,7 +553,7 @@
public boolean valueSatisfy(${filter.dataType} value) {
<#if filter.dataType == "boolean">
return Boolean.compare(constant,value) <= 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return constant.compareTo(value) <= 0;
<#else>
return constant <= value;
@@ -557,7 +563,7 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -572,14 +578,14 @@
return false;
}
// drop if value > max
- return constant > (${filter.javaBoxName}) statistics.getMaxValue();
+ return constant > (${javaClassName}) statistics.getMaxValue();
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -592,7 +598,7 @@
if(statistics.isEmpty()){
return false;
}
- return constant <= (${filter.javaBoxName}) statistics.getMinValue();
+ return constant <= (${javaClassName}) statistics.getMinValue();
</#if>
}
@@ -615,7 +621,7 @@
protected ValueColumnRangeFilter(int measurementIndex, ${filter.dataType} min, ${filter.dataType} max) {
super(measurementIndex);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.min = Objects.requireNonNull(min,"min cannot be null");
this.max = Objects.requireNonNull(max,"max cannot be null");
<#else>
@@ -627,7 +633,7 @@
@SuppressWarnings("unchecked")
protected ValueColumnRangeFilter(ByteBuffer buffer) {
super(buffer);
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
this.min = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer),"min cannot be null");
this.max = Objects.requireNonNull(ReadWriteIOUtils.read${filter.dataType?cap_first}(buffer),"max cannot be null");
<#else>
@@ -655,7 +661,7 @@
return false;
}
ValueColumnRangeFilter that = (ValueColumnRangeFilter) o;
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return min.equals(that.min) && max.equals(that.max);
<#else>
return min == that.min && max == that.max;
@@ -696,7 +702,7 @@
<#if filter.dataType == "boolean">
return Boolean.compare(min,value) <= 0
&& Boolean.compare(max,value) >= 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return min.compareTo(value) <= 0
&& max.compareTo(value) >= 0;
<#else>
@@ -707,7 +713,7 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -722,15 +728,15 @@
if(statistics.isEmpty()){
return false;
}
- return (${filter.javaBoxName}) statistics.getMaxValue() < min
- || (${filter.javaBoxName}) statistics.getMinValue() > max;
+ return (${javaClassName}) statistics.getMaxValue() < min
+ || (${javaClassName}) statistics.getMinValue() > max;
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -744,8 +750,8 @@
if(statistics.isEmpty()){
return false;
}
- return (${filter.javaBoxName}) statistics.getMinValue() >= min
- && (${filter.javaBoxName}) statistics.getMaxValue() <= max;
+ return (${javaClassName}) statistics.getMinValue() >= min
+ && (${javaClassName}) statistics.getMaxValue() <= max;
</#if>
}
@@ -781,7 +787,7 @@
<#if filter.dataType == "boolean">
return Boolean.compare(min,value) > 0
|| Boolean.compare(max,value) < 0;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
return min.compareTo(value) > 0 || max.compareTo(value) < 0;
<#else>
return min > value || max < value;
@@ -791,7 +797,7 @@
@Override
@SuppressWarnings("unchecked")
public boolean canSkip(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -806,15 +812,15 @@
if(statistics.isEmpty()){
return false;
}
- return (${filter.javaBoxName}) statistics.getMinValue() >= min
- && (${filter.javaBoxName}) statistics.getMaxValue() <= max;
+ return (${javaClassName}) statistics.getMinValue() >= min
+ && (${javaClassName}) statistics.getMaxValue() <= max;
</#if>
}
@Override
@SuppressWarnings("unchecked")
public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
- <#if filter.dataType == "boolean" || filter.dataType == "Binary">
+ <#if filter.dataType == "boolean" || filter.dataType == "Binary" || filter.dataType == "String">
<#if filter.javaBoxName == "String">
if(statistics.isEmpty()){
return false;
@@ -828,8 +834,8 @@
if(statistics.isEmpty()){
return false;
}
- return (${filter.javaBoxName}) statistics.getMinValue() > max
- || (${filter.javaBoxName}) statistics.getMaxValue() < min;
+ return (${javaClassName}) statistics.getMinValue() > max
+ || (${javaClassName}) statistics.getMaxValue() < min;
</#if>
}
@@ -850,7 +856,7 @@
<#if filter.javaBoxName == "String">
protected final Set<${filter.dataType}> candidates;
<#else>
- protected final Set<${filter.javaBoxName}> candidates;
+ protected final Set<${javaClassName}> candidates;
</#if>
protected final ${filter.dataType} candidatesMin;
@@ -859,7 +865,7 @@
<#if filter.javaBoxName == "String">
protected ValueColumnSetFilter(int measurementIndex, Set<${filter.dataType}> candidates) {
<#else>
- protected ValueColumnSetFilter(int measurementIndex, Set<${filter.javaBoxName}> candidates) {
+ protected ValueColumnSetFilter(int measurementIndex, Set<${javaClassName}> candidates) {
</#if>
super(measurementIndex);
this.candidates = candidates;
@@ -867,18 +873,18 @@
<#if filter.javaBoxName == "String">
Set<${filter.dataType}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet());
<#else>
- Set<${filter.javaBoxName}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet());
+ Set<${javaClassName}> filteredSet = candidates.stream().filter(Objects::nonNull).collect(Collectors.toSet());
</#if>
<#if filter.dataType == "boolean">
// BooleanStatistics is not available
this.candidatesMin = false;
this.candidatesMax = false;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : null;
this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : null;
<#else>
- this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : ${filter.javaBoxName}.MIN_VALUE;
- this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : ${filter.javaBoxName}.MAX_VALUE;
+ this.candidatesMin = !filteredSet.isEmpty() ? Collections.min(filteredSet) : ${javaClassName}.MIN_VALUE;
+ this.candidatesMax = !filteredSet.isEmpty() ? Collections.max(filteredSet) : ${javaClassName}.MAX_VALUE;
</#if>
}
@@ -888,18 +894,18 @@
<#if filter.javaBoxName == "String">
this.candidates = ReadWriteIOUtils.read${filter.dataType}Set(buffer);
<#else>
- this.candidates = ReadWriteIOUtils.read${filter.javaBoxName}Set(buffer);
+ this.candidates = ReadWriteIOUtils.read${javaClassName}Set(buffer);
</#if>
<#if filter.dataType == "boolean">
// BooleanStatistics is not available
this.candidatesMin = false;
this.candidatesMax = false;
- <#elseif filter.dataType == "Binary">
+ <#elseif filter.dataType == "Binary" || filter.dataType == "String">
this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : null;
this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : null;
<#else>
- this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : ${filter.javaBoxName}.MAX_VALUE;
- this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : ${filter.javaBoxName}.MAX_VALUE;
+ this.candidatesMin = !candidates.isEmpty() ? Collections.min(candidates) : ${javaClassName}.MAX_VALUE;
+ this.candidatesMax = !candidates.isEmpty() ? Collections.max(candidates) : ${javaClassName}.MAX_VALUE;
</#if>
if(hasNull){
this.candidates.add(null);
@@ -913,7 +919,7 @@
<#if filter.javaBoxName == "String">
ReadWriteIOUtils.write${filter.dataType}Set(candidates, outputStream);
<#else>
- ReadWriteIOUtils.write${filter.javaBoxName}Set(candidates, outputStream);
+ ReadWriteIOUtils.write${javaClassName}Set(candidates, outputStream);
</#if>
}
@@ -951,7 +957,7 @@
super(measurementIndex, candidates);
}
<#else>
- public ValueIn(int measurementIndex, Set<${filter.javaBoxName}> candidates) {
+ public ValueIn(int measurementIndex, Set<${javaClassName}> candidates) {
super(measurementIndex, candidates);
}
</#if>
@@ -962,7 +968,7 @@
@Override
public boolean valueSatisfy(Object value){
- return candidates.contains((${filter.javaBoxName}) value);
+ return candidates.contains((${javaClassName}) value);
}
@Override
@@ -990,12 +996,12 @@
if (statistics.isPresent()) {
Statistics<? extends Serializable> stat = statistics.get();
- <#if filter.dataType == "Binary" && filter.javaBoxName == "String">
+ <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")>
${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue();
${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue();
<#else>
- ${filter.javaBoxName} valuesMin = (${filter.javaBoxName}) stat.getMinValue();
- ${filter.javaBoxName} valuesMax = (${filter.javaBoxName}) stat.getMaxValue();
+ ${javaClassName} valuesMin = (${javaClassName}) stat.getMinValue();
+ ${javaClassName} valuesMax = (${javaClassName}) stat.getMaxValue();
</#if>
// All values are same
if (valuesMin.equals(valuesMax)) {
@@ -1003,7 +1009,7 @@
} else {
if (!candidates.isEmpty()) {
// All values are less than min, or greater than max
- <#if filter.dataType == "Binary" && filter.javaBoxName == "String">
+ <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")>
return candidatesMin.compareTo(valuesMax) > 0
|| candidatesMax.compareTo(valuesMin) < 0;
<#else>
@@ -1044,12 +1050,12 @@
// All values are same
if (statistics.isPresent()) {
Statistics<? extends Serializable> stat = statistics.get();
- <#if filter.dataType == "Binary" && filter.javaBoxName == "String">
+ <#if (filter.dataType == "Binary" && filter.javaBoxName == "String") || (filter.dataType == "String" && filter.javaBoxName == "Tag")>
${filter.dataType} valuesMin = (${filter.dataType}) stat.getMinValue();
${filter.dataType} valuesMax = (${filter.dataType}) stat.getMaxValue();
<#else>
- ${filter.javaBoxName} valuesMin = (${filter.javaBoxName}) stat.getMinValue();
- ${filter.javaBoxName} valuesMax = (${filter.javaBoxName}) stat.getMaxValue();
+ ${javaClassName} valuesMin = (${javaClassName}) stat.getMinValue();
+ ${javaClassName} valuesMax = (${javaClassName}) stat.getMaxValue();
</#if>
// All values are same
if (valuesMin.equals(valuesMax)) {
@@ -1095,7 +1101,7 @@
super(measurementIndex, candidates);
}
<#else>
- public ValueNotIn(int measurementIndex, Set<${filter.javaBoxName}> candidates) {
+ public ValueNotIn(int measurementIndex, Set<${javaClassName}> candidates) {
super(measurementIndex, candidates);
}
</#if>
@@ -1106,7 +1112,7 @@
@Override
public boolean valueSatisfy(Object value){
- return !candidates.contains((${filter.javaBoxName}) value);
+ return !candidates.contains((${javaClassName}) value);
}
@Override
@@ -1203,7 +1209,7 @@
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find();
<#else>
return pattern.matcher(new MatcherInput(String.valueOf(value), new AccessCount())).find();
@@ -1248,7 +1254,7 @@
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return !pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find();
<#else>
return !pattern.matcher(new MatcherInput(String.valueOf(value), new AccessCount())).find();
@@ -1353,7 +1359,7 @@
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return pattern.getMatcher().match(value.toString().getBytes());
<#else>
return pattern.getMatcher().match(String.valueOf(value).getBytes());
@@ -1398,7 +1404,7 @@
@Override
public boolean valueSatisfy(${filter.dataType} value) {
- <#if filter.dataType == "Binary">
+ <#if filter.dataType == "Binary" || filter.dataType == "String">
return !pattern.getMatcher().match(value.toString().getBytes());
<#else>
return !pattern.getMatcher().match(String.valueOf(value).getBytes());
diff --git a/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl b/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl
index e1fd9fe..d057484 100644
--- a/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl
+++ b/java/tsfile/src/main/codegen/templates/FilterTemplate.ftl
@@ -66,6 +66,7 @@
protected abstract boolean valueSatisfy(${filter.dataType} value);
+ <#if filter.javaBoxName != "Tag">
@Override
public boolean[] satisfyTsBlock(boolean[] selection, TsBlock tsBlock) {
Column valueColumn = tsBlock.getValueColumns()[measurementIndex];
@@ -83,6 +84,12 @@
}
return satisfyInfo;
}
+ <#else >
+ @Override
+ public boolean[] satisfyTsBlock(boolean[] selection, TsBlock tsBlock) {
+ throw new IllegalArgumentException("TagFilter cannot be applied to TsBlock");
+ }
+ </#if>
@Override
public void serialize(DataOutputStream outputStream) throws IOException {
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
index 4445bf9..74c99ef 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/DeviceMetaIterator.java
@@ -24,7 +24,7 @@
import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.tsfile.read.TsFileSequenceReader;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
@@ -43,15 +43,15 @@
private final TsFileSequenceReader tsFileSequenceReader;
private final Queue<MetadataIndexNode> metadataIndexNodes = new ArrayDeque<>();
private final Queue<Pair<IDeviceID, MetadataIndexNode>> resultCache = new ArrayDeque<>();
- private final ExpressionTree idFilter;
+ private final Filter tagFilter;
public DeviceMetaIterator(
TsFileSequenceReader tsFileSequenceReader,
MetadataIndexNode metadataIndexNode,
- ExpressionTree idFilter) {
+ Filter tagFilter) {
this.tsFileSequenceReader = tsFileSequenceReader;
this.metadataIndexNodes.add(metadataIndexNode);
- this.idFilter = idFilter;
+ this.tagFilter = tagFilter;
}
@Override
@@ -74,7 +74,8 @@
for (int i = 0; i < leafChildren.size(); i++) {
IMetadataIndexEntry child = leafChildren.get(i);
final IDeviceID deviceID = ((DeviceMetadataIndexEntry) child).getDeviceID();
- if (idFilter != null && !idFilter.satisfy(deviceID)) {
+ // time is only a placeholder here
+ if (tagFilter != null && !tagFilter.satisfyRow(0, deviceID.getSegments())) {
continue;
}
@@ -114,6 +115,7 @@
if (!resultCache.isEmpty()) {
return;
}
+ break;
case INTERNAL_DEVICE:
loadInternalNode(currentNode);
break;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
index 3503e49..3f74178 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/IMetadataQuerier.java
@@ -28,7 +28,7 @@
import org.apache.tsfile.file.metadata.TsFileMetadata;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.TimeRange;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import java.io.IOException;
@@ -87,5 +87,5 @@
void clear();
Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator(
- MetadataIndexNode root, ExpressionTree idFilter);
+ MetadataIndexNode root, Filter idFilter);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
index ae7fb9e..68e1953 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/controller/MetadataQuerierByFileImpl.java
@@ -34,7 +34,7 @@
import org.apache.tsfile.read.TsFileSequenceReader.LocateStatus;
import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.TimeRange;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
import java.io.IOException;
@@ -306,7 +306,7 @@
@Override
public Iterator<Pair<IDeviceID, MetadataIndexNode>> deviceIterator(
- MetadataIndexNode root, ExpressionTree idFilter) {
- return new DeviceMetaIterator(tsFileReader, root, idFilter);
+ MetadataIndexNode root, Filter tagFilter) {
+ return new DeviceMetaIterator(tsFileReader, root, tagFilter);
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java
index cc48210..a5aac76 100755
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/Filter.java
@@ -77,6 +77,8 @@
public abstract boolean satisfyBinary(long time, Binary value);
+ public abstract boolean satisfyString(long time, String value);
+
/**
* To examine whether the row(with time and values) is satisfied with the filter.
*
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java
index 18d444d..a23ee31 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/TimeFilter.java
@@ -75,6 +75,11 @@
}
@Override
+ public boolean satisfyString(long time, String value) {
+ return timeSatisfy(time);
+ }
+
+ @Override
public boolean satisfyRow(long time, Object[] values) {
// only use time to filter
return timeSatisfy(time);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java
index e1ae98b..d715352 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/basic/ValueFilter.java
@@ -85,13 +85,18 @@
}
@Override
+ public boolean satisfyString(long time, String value) {
+ throw new UnSupportedDataTypeException(getClass().getName());
+ }
+
+ @Override
public boolean satisfyBinary(long time, Binary value) {
throw new UnSupportedDataTypeException(getClass().getName());
}
@Override
public boolean satisfyRow(long time, Object[] values) {
- return satisfy(time, values[measurementIndex]);
+ return satisfy(time, measurementIndex < values.length ? values[measurementIndex] : null);
}
@Override
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java
new file mode 100644
index 0000000..a315950
--- /dev/null
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/factory/TagFilterBuilder.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.read.filter.factory;
+
+import org.apache.tsfile.annotations.TsFileApi;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.filter.basic.Filter;
+import org.apache.tsfile.read.filter.operator.TagFilterOperators.ValueEq;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+
+public class TagFilterBuilder {
+ private TableSchema tableSchema;
+
+ public TagFilterBuilder(TableSchema tableSchema) {
+ this.tableSchema = tableSchema;
+ }
+
+ @TsFileApi
+ public Filter eq(String columnName, Object value) {
+ int idColumnOrder = tableSchema.findIdColumnOrder(columnName);
+ if (idColumnOrder == -1) {
+ throw new IllegalArgumentException("Column '" + columnName + "' is not a tag column");
+ }
+ IMeasurementSchema columnSchema = tableSchema.findColumnSchema(columnName);
+
+ // +1 for table name
+ return new ValueEq(idColumnOrder + 1, (String) value);
+ }
+}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java
index de0f338..8df4e50 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/And.java
@@ -78,6 +78,11 @@
}
@Override
+ public boolean satisfyString(long time, String value) {
+ return left.satisfyString(time, value) && right.satisfyString(time, value);
+ }
+
+ @Override
public boolean satisfyRow(long time, Object[] values) {
return left.satisfyRow(time, values) && right.satisfyRow(time, values);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java
index 0091891..e02a369 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Not.java
@@ -91,6 +91,11 @@
}
@Override
+ public boolean satisfyString(long time, String value) {
+ return !filter.satisfyString(time, value);
+ }
+
+ @Override
public boolean satisfyBooleanRow(long time, boolean[] values) {
return !filter.satisfyBooleanRow(time, values);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java
index f0f1c2f..97a7875 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/Or.java
@@ -78,6 +78,11 @@
}
@Override
+ public boolean satisfyString(long time, String value) {
+ return left.satisfyString(time, value) || right.satisfyString(time, value);
+ }
+
+ @Override
public boolean satisfyRow(long time, Object[] values) {
return left.satisfyRow(time, values) || right.satisfyRow(time, values);
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java
index 0f2d72b..2e5d070 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/ResultSet.java
@@ -20,9 +20,11 @@
package org.apache.tsfile.read.query.dataset;
import org.apache.tsfile.annotations.TsFileApi;
+import org.apache.tsfile.write.record.TSRecord;
import java.io.IOException;
import java.time.LocalDate;
+import java.util.Iterator;
public interface ResultSet extends AutoCloseable {
@@ -87,5 +89,8 @@
boolean isNull(int columnIndex);
@TsFileApi
- public abstract void close();
+ void close();
+
+ @TsFileApi
+ Iterator<TSRecord> recordIterator();
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java
index 5e05077..8a585df 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TableResultSet.java
@@ -26,27 +26,36 @@
import org.apache.tsfile.read.common.block.TsBlock;
import org.apache.tsfile.read.reader.IPointReader;
import org.apache.tsfile.read.reader.block.TsBlockReader;
+import org.apache.tsfile.write.record.TSRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Iterator;
import java.util.List;
+import java.util.NoSuchElementException;
public class TableResultSet extends AbstractResultSet {
+
private static final Logger LOG = LoggerFactory.getLogger(TableResultSet.class);
private TsBlockReader tsBlockReader;
private IPointReader tsBlockPointReader;
private List<String> columnNameList;
private List<TSDataType> dataTypeList;
+ private String tableName;
public TableResultSet(
- TsBlockReader tsBlockReader, List<String> columnNameList, List<TSDataType> dataTypeList) {
+ TsBlockReader tsBlockReader,
+ List<String> columnNameList,
+ List<TSDataType> dataTypeList,
+ String tableName) {
super(columnNameList, dataTypeList);
this.tsBlockReader = tsBlockReader;
this.columnNameList = columnNameList;
this.dataTypeList = dataTypeList;
+ this.tableName = tableName;
}
@Override
@@ -85,4 +94,89 @@
LOG.error("Failed to close tsBlockReader");
}
}
+
+ @Override
+ public Iterator<TSRecord> recordIterator() {
+ return new RecordIterator();
+ }
+
+ private class RecordIterator implements Iterator<TSRecord> {
+
+ private TSRecord cachedRecord = null;
+ private boolean exhausted = false;
+
+ @Override
+ public boolean hasNext() {
+ if (cachedRecord != null) {
+ return true;
+ }
+ if (exhausted) {
+ return false;
+ }
+
+ try {
+ return cacheNextRecord();
+ } catch (IOException e) {
+ throw new NoSuchElementException(e.toString());
+ }
+ }
+
+ private boolean cacheNextRecord() throws IOException {
+ boolean next = TableResultSet.this.next();
+ if (!next) {
+ exhausted = true;
+ return false;
+ }
+
+ cachedRecord = new TSRecord(tableName, getLong("Time"));
+ for (int i = 0; i < columnNameList.size(); i++) {
+ Field field = currentRow.getFields().get(i);
+ if (field != null) {
+ switch (field.getDataType()) {
+ case INT32:
+ case DATE:
+ cachedRecord.addPoint(columnNameList.get(i), field.getIntV());
+ break;
+ case INT64:
+ case TIMESTAMP:
+ cachedRecord.addPoint(columnNameList.get(i), field.getLongV());
+ break;
+ case FLOAT:
+ cachedRecord.addPoint(columnNameList.get(i), field.getFloatV());
+ break;
+ case DOUBLE:
+ cachedRecord.addPoint(columnNameList.get(i), field.getDoubleV());
+ break;
+ case STRING:
+ case TEXT:
+ cachedRecord.addPoint(columnNameList.get(i), field.getStringValue());
+ break;
+ case BLOB:
+ cachedRecord.addPoint(columnNameList.get(i), field.getBinaryV().getValues());
+ break;
+ case BOOLEAN:
+ cachedRecord.addPoint(columnNameList.get(i), field.getBoolV());
+ break;
+ case VECTOR:
+ case UNKNOWN:
+ default:
+ break;
+ }
+ } else {
+ cachedRecord.dataPointList.add(null);
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public TSRecord next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ TSRecord ret = cachedRecord;
+ cachedRecord = null;
+ return ret;
+ }
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java
index 0f3aade..1f12cc6 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/dataset/TreeResultSet.java
@@ -21,8 +21,10 @@
import org.apache.tsfile.annotations.TsFileApi;
import org.apache.tsfile.read.common.Path;
+import org.apache.tsfile.write.record.TSRecord;
import java.io.IOException;
+import java.util.Iterator;
import java.util.stream.Collectors;
public class TreeResultSet extends AbstractResultSet {
@@ -51,4 +53,9 @@
public void close() {
// nothing to be done
}
+
+ @Override
+ public Iterator<TSRecord> recordIterator() {
+ return null;
+ }
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
index a192a36..c50d1bf 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/TableQueryExecutor.java
@@ -29,6 +29,7 @@
import org.apache.tsfile.read.controller.IChunkLoader;
import org.apache.tsfile.read.controller.IMetadataQuerier;
import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.executor.task.DeviceTaskIterator;
import org.apache.tsfile.read.reader.block.DeviceOrderedTsBlockReader;
import org.apache.tsfile.read.reader.block.TsBlockReader;
@@ -62,7 +63,7 @@
* @param tableName table to query
* @param columns columns to query (ID or MEASUREMENT)
* @param timeFilter time predicate
- * @param idFilter id predicate
+ * @param tagFilter id predicate
* @param measurementFilter measurement predicate
* @return an iterator of TsBlocks
* @throws ReadProcessException if the read process fails
@@ -71,7 +72,7 @@
String tableName,
List<String> columns,
ExpressionTree timeFilter,
- ExpressionTree idFilter,
+ Filter tagFilter,
ExpressionTree measurementFilter)
throws ReadProcessException {
TsFileMetadata fileMetadata = metadataQuerier.getWholeFileMetadata();
@@ -90,7 +91,7 @@
DeviceTaskIterator deviceTaskIterator =
new DeviceTaskIterator(
- columns, tableRoot, columnMapping, metadataQuerier, idFilter, tableSchema);
+ columns, tableRoot, columnMapping, metadataQuerier, tagFilter, tableSchema);
switch (tableQueryOrdering) {
case DEVICE:
return new DeviceOrderedTsBlockReader(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
index 8ee4465..399cd32 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/query/executor/task/DeviceTaskIterator.java
@@ -23,7 +23,7 @@
import org.apache.tsfile.file.metadata.MetadataIndexNode;
import org.apache.tsfile.file.metadata.TableSchema;
import org.apache.tsfile.read.controller.IMetadataQuerier;
-import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.executor.TableQueryExecutor.ColumnMapping;
import org.apache.tsfile.utils.Pair;
@@ -41,11 +41,11 @@
MetadataIndexNode indexRoot,
ColumnMapping columnMapping,
IMetadataQuerier metadataQuerier,
- ExpressionTree idFilter,
+ Filter tagFilter,
TableSchema tableSchema) {
this.columnNames = columnNames;
this.columnMapping = columnMapping;
- this.deviceMetaIterator = metadataQuerier.deviceIterator(indexRoot, idFilter);
+ this.deviceMetaIterator = metadataQuerier.deviceIterator(indexRoot, tagFilter);
this.tableSchema = tableSchema;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
index 894353f..927ac89 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/DeviceTableModelReader.java
@@ -31,6 +31,7 @@
import org.apache.tsfile.read.controller.IMetadataQuerier;
import org.apache.tsfile.read.controller.MetadataQuerierByFileImpl;
import org.apache.tsfile.read.expression.ExpressionTree;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.dataset.ResultSet;
import org.apache.tsfile.read.query.dataset.TableResultSet;
import org.apache.tsfile.read.query.executor.TableQueryExecutor;
@@ -79,6 +80,13 @@
@TsFileApi
public ResultSet query(String tableName, List<String> columnNames, long startTime, long endTime)
throws IOException, NoTableException, NoMeasurementException, ReadProcessException {
+ return query(tableName, columnNames, startTime, endTime, null);
+ }
+
+ @Override
+ public ResultSet query(
+ String tableName, List<String> columnNames, long startTime, long endTime, Filter tagFilter)
+ throws ReadProcessException, IOException, NoTableException, NoMeasurementException {
String lowerCaseTableName = tableName.toLowerCase();
TableSchema tableSchema = fileReader.getTableSchemaMap().get(lowerCaseTableName);
if (tableSchema == null) {
@@ -100,9 +108,9 @@
lowerCaseTableName,
lowerCaseColumnNames,
new ExpressionTree.TimeBetweenAnd(startTime, endTime),
- null,
+ tagFilter,
null);
- return new TableResultSet(tsBlockReader, columnNames, dataTypeList);
+ return new TableResultSet(tsBlockReader, columnNames, dataTypeList, tableName);
}
@Override
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
index 885e845..995f73f 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/v4/ITsFileReader.java
@@ -24,6 +24,7 @@
import org.apache.tsfile.exception.write.NoMeasurementException;
import org.apache.tsfile.exception.write.NoTableException;
import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.query.dataset.ResultSet;
import java.io.IOException;
@@ -37,6 +38,11 @@
throws ReadProcessException, IOException, NoTableException, NoMeasurementException;
@TsFileApi
+ ResultSet query(
+ String tableName, List<String> columnNames, long startTime, long endTime, Filter tagFilter)
+ throws ReadProcessException, IOException, NoTableException, NoMeasurementException;
+
+ @TsFileApi
Optional<TableSchema> getTableSchemas(String tableName) throws IOException;
@TsFileApi
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
index 6e38e51..e6b72e4 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/ReadWriteIOUtils.java
@@ -997,6 +997,18 @@
return set;
}
+ public static Set<String> readStringSet(ByteBuffer buffer) {
+ int size = readInt(buffer);
+ if (size <= 0) {
+ return Collections.emptySet();
+ }
+ Set<String> set = new HashSet<>();
+ for (int i = 0; i < size; i++) {
+ set.add(readString(buffer));
+ }
+ return set;
+ }
+
// read object set with self define length
public static <T> Set<T> readObjectSet(ByteBuffer buffer) {
int size = readInt(buffer);
@@ -1082,6 +1094,16 @@
}
}
+ public static void writeStringSet(Set<String> set, DataOutputStream outputStream)
+ throws IOException {
+ write(set.contains(null) ? set.size() - 1 : set.size(), outputStream);
+ for (String e : set) {
+ if (e != null) {
+ write(e, outputStream);
+ }
+ }
+ }
+
public static CompressionType readCompressionType(InputStream inputStream) throws IOException {
byte n = readByte(inputStream);
return CompressionType.deserialize(n);
@@ -1145,6 +1167,7 @@
BINARY,
BOOLEAN,
STRING,
+ TAG,
NULL
}
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
index a9f8766..090d7f8 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/read/query/ResultSetTest.java
@@ -22,10 +22,12 @@
import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.read.filter.factory.TagFilterBuilder;
import org.apache.tsfile.read.query.dataset.ResultSet;
import org.apache.tsfile.read.query.dataset.ResultSetMetadata;
import org.apache.tsfile.read.v4.DeviceTableModelReader;
import org.apache.tsfile.utils.TsFileGeneratorForTest;
+import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.apache.tsfile.write.v4.ITsFileWriter;
@@ -40,6 +42,7 @@
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
+import java.util.Iterator;
public class ResultSetTest {
@@ -140,6 +143,135 @@
}
@Test
+ public void testQueryTableWithTagFilter() throws Exception {
+ TableSchema tableSchema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(
+ new MeasurementSchema("id1", TSDataType.STRING),
+ new MeasurementSchema("id2", TSDataType.STRING),
+ new MeasurementSchema("s1", TSDataType.BOOLEAN),
+ new MeasurementSchema("s2", TSDataType.BOOLEAN)),
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ Tablet tablet =
+ new Tablet(
+ Arrays.asList("id1", "id2", "s1", "s2"),
+ Arrays.asList(
+ TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN),
+ 1024);
+ tablet.addTimestamp(0, 0);
+ tablet.addValue("id1", 0, "id_field1");
+ tablet.addValue("id2", 0, "id_field2");
+ tablet.addValue("s1", 0, true);
+ tablet.addValue("s2", 0, false);
+
+ tablet.addTimestamp(1, 1);
+ tablet.addValue("id1", 1, "id_field1_2");
+ tablet.addValue("s2", 1, true);
+
+ tablet.addTimestamp(2, 2);
+
+ try (ITsFileWriter writer =
+ new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) {
+ writer.write(tablet);
+ }
+
+ // eq
+ TagFilterBuilder filterBuilder = new TagFilterBuilder(tableSchema);
+ try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile);
+ ResultSet resultSet =
+ tsFileReader.query(
+ "T1",
+ Arrays.asList("ID1", "ID2", "S2", "S1"),
+ 0,
+ 2,
+ filterBuilder.eq("ID1", "id_field1"))) {
+
+ Assert.assertTrue(resultSet.next());
+ Assert.assertEquals(0, resultSet.getLong(1));
+ Assert.assertEquals("id_field1", resultSet.getString(2));
+ Assert.assertEquals("id_field2", resultSet.getString(3));
+ Assert.assertFalse(resultSet.getBoolean(4));
+ Assert.assertTrue(resultSet.getBoolean(5));
+ }
+ }
+
+ @Test
+ public void testQueryTableByIterator() throws Exception {
+ TableSchema tableSchema =
+ new TableSchema(
+ "t1",
+ Arrays.asList(
+ new MeasurementSchema("id1", TSDataType.STRING),
+ new MeasurementSchema("id2", TSDataType.STRING),
+ new MeasurementSchema("s1", TSDataType.BOOLEAN),
+ new MeasurementSchema("s2", TSDataType.BOOLEAN)),
+ Arrays.asList(
+ ColumnCategory.TAG,
+ ColumnCategory.TAG,
+ ColumnCategory.FIELD,
+ ColumnCategory.FIELD));
+ Tablet tablet =
+ new Tablet(
+ Arrays.asList("id1", "id2", "s1", "s2"),
+ Arrays.asList(
+ TSDataType.STRING, TSDataType.STRING, TSDataType.BOOLEAN, TSDataType.BOOLEAN),
+ 1024);
+ tablet.addTimestamp(0, 0);
+ tablet.addValue("id1", 0, "id_field1");
+ tablet.addValue("id2", 0, "id_field2");
+ tablet.addValue("s1", 0, true);
+ tablet.addValue("s2", 0, false);
+
+ tablet.addTimestamp(1, 1);
+ tablet.addValue("id1", 1, "id_field1_2");
+ tablet.addValue("s2", 1, true);
+
+ tablet.addTimestamp(2, 2);
+
+ try (ITsFileWriter writer =
+ new TsFileWriterBuilder().file(tsfile).tableSchema(tableSchema).build()) {
+ writer.write(tablet);
+ }
+
+ try (DeviceTableModelReader tsFileReader = new DeviceTableModelReader(tsfile);
+ ResultSet resultSet =
+ tsFileReader.query("T1", Arrays.asList("ID1", "ID2", "S2", "S1"), 0, 2); ) {
+ Iterator<TSRecord> tsRecordIterator = resultSet.recordIterator();
+
+ Assert.assertTrue(tsRecordIterator.hasNext());
+ TSRecord tsRecord = tsRecordIterator.next();
+ Assert.assertEquals(2, tsRecord.time);
+ Assert.assertNull(tsRecord.dataPointList.get(0));
+ Assert.assertNull(tsRecord.dataPointList.get(1));
+ Assert.assertNull(tsRecord.dataPointList.get(2));
+ Assert.assertNull(tsRecord.dataPointList.get(3));
+
+ Assert.assertTrue(tsRecordIterator.hasNext());
+ tsRecord = tsRecordIterator.next();
+ Assert.assertEquals(0, tsRecord.time);
+ Assert.assertEquals("id_field1", tsRecord.dataPointList.get(0).getValue().toString());
+ Assert.assertEquals("id_field2", tsRecord.dataPointList.get(1).getValue().toString());
+ Assert.assertFalse((Boolean) tsRecord.dataPointList.get(2).getValue());
+ Assert.assertTrue((Boolean) tsRecord.dataPointList.get(3).getValue());
+
+ Assert.assertTrue(tsRecordIterator.hasNext());
+ tsRecord = tsRecordIterator.next();
+ Assert.assertEquals(1, tsRecord.time);
+ Assert.assertEquals("id_field1_2", tsRecord.dataPointList.get(0).getValue().toString());
+ Assert.assertNull(tsRecord.dataPointList.get(1));
+ Assert.assertTrue((Boolean) tsRecord.dataPointList.get(2).getValue());
+ Assert.assertNull(tsRecord.dataPointList.get(3));
+
+ Assert.assertFalse(tsRecordIterator.hasNext());
+ }
+ }
+
+ @Test
public void testQueryWithMaxValue() throws Exception {
TableSchema tableSchema =
new TableSchema(
