set druid.expressions.useStrictBooleans to true by default (#14734)
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
index 80d208c..6f96a2c 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
@@ -84,7 +84,7 @@
static {
NullHandling.initializeForTests();
- ExpressionProcessing.initializeForStrictBooleansTests(true);
+ ExpressionProcessing.initializeForTests();
}
private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig()
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
index da42aae..fe5dfff 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
@@ -90,7 +90,7 @@
static {
NullHandling.initializeForTests();
- ExpressionProcessing.initializeForStrictBooleansTests(true);
+ ExpressionProcessing.initializeForTests();
}
private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig()
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 362e2d5..cb4ee2e 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -2288,8 +2288,8 @@
|Key|Description|Default|
|---|-----------|-------|
-|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values will be either a `1` or `0`. See [expression documentation](../querying/math-expr.md#logical-operator-modes)|false|
-|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays.|false|
+|`druid.expressions.useStrictBooleans`|Controls the behavior of Druid boolean operators and functions, if set to `true` all boolean values are either `1` or `0`. See [expression documentation](../querying/math-expr.md#logical-operator-modes) for more information.|true|
+|`druid.expressions.allowNestedArrays`|If enabled, Druid array expressions can create nested arrays.|true|
### Router
#### Router Process Configs
diff --git a/docs/development/experimental-features.md b/docs/development/experimental-features.md
index d33f634..36c7282 100644
--- a/docs/development/experimental-features.md
+++ b/docs/development/experimental-features.md
@@ -57,4 +57,3 @@
- [Configuration reference](../configuration/index.md)
- `CLOSED_SEGMENTS_SINKS` mode
- - Expression processing configuration `druid.expressions.allowNestedArrays`
diff --git a/docs/ingestion/schema-design.md b/docs/ingestion/schema-design.md
index 556cdc4..0749d20 100644
--- a/docs/ingestion/schema-design.md
+++ b/docs/ingestion/schema-design.md
@@ -258,8 +258,8 @@
When performing type-aware schema discovery, Druid can discover all of the columns of your input data (that aren't in
the exclusion list). Druid automatically chooses the most appropriate native Druid type among `STRING`, `LONG`,
`DOUBLE`, `ARRAY<STRING>`, `ARRAY<LONG>`, `ARRAY<DOUBLE>`, or `COMPLEX<json>` for nested data. For input formats with
-native boolean types, Druid ingests these values as strings if `druid.expressions.useStrictBooleans` is set to `false`
-(the default), or longs if set to `true` (for more SQL compatible behavior). Array typed columns can be queried using
+native boolean types, Druid ingests these values as longs if `druid.expressions.useStrictBooleans` is set to `true`
+(the default) or strings if set to `false`. Array typed columns can be queried using
the [array functions](../querying/sql-array-functions.md) or [UNNEST](../querying/sql-functions.md#unnest). Nested
columns can be queried with the [JSON functions](../querying/sql-json-functions.md).
diff --git a/docs/querying/math-expr.md b/docs/querying/math-expr.md
index 3da1fd3..dcb4b7c 100644
--- a/docs/querying/math-expr.md
+++ b/docs/querying/math-expr.md
@@ -299,7 +299,7 @@
* constants and identifiers are supported for any column type
* `cast` is supported for numeric and string types
* math operators: `+`,`-`,`*`,`/`,`%`,`^` are supported for numeric types
-* logical operators: `!`, `&&`, `||`, are supported for string and numeric types (if `druid.expressions.useStrictBooleans=true`)
+* logical operators: `!`, `&&`, `||`, are supported for string and numeric types
* comparison operators: `=`, `!=`, `>`, `>=`, `<`, `<=` are supported for string and numeric types
* math functions: `abs`, `acos`, `asin`, `atan`, `cbrt`, `ceil`, `cos`, `cosh`, `cot`, `exp`, `expm1`, `floor`, `getExponent`, `log`, `log10`, `log1p`, `nextUp`, `rint`, `signum`, `sin`, `sinh`, `sqrt`, `tan`, `tanh`, `toDegrees`, `toRadians`, `ulp`, `atan2`, `copySign`, `div`, `hypot`, `max`, `min`, `nextAfter`, `pow`, `remainder`, `scalb` are supported for numeric types
* time functions: `timestamp_floor` (with constant granularity argument) is supported for numeric types
@@ -309,9 +309,7 @@
* other: `parse_long` is supported for numeric and string types
## Logical operator modes
-Prior to the 0.23 release of Apache Druid, boolean function expressions have inconsistent handling of true and false values, and the logical 'and' and 'or' operators behave in a manner that is incompatible with SQL, even if SQL compatible null handling mode (`druid.generic.useDefaultValueForNull=false`) is enabled. Logical operators also pass through their input values similar to many scripting languages, and treat `null` as false, which can result in some rather strange behavior. Other boolean operations, such as comparisons and equality, retain their input types (e.g. `DOUBLE` comparison would produce `1.0` for true and `0.0` for false), while many other boolean functions strictly produce `LONG` typed values of `1` for true and `0` for false.
-
-After 0.23, while the inconsistent legacy behavior is still the default, it can be optionally be changed by setting `druid.expressions.useStrictBooleans=true`, so that these operations will allow correctly treating `null` values as "unknown" for SQL compatible behavior, and _all boolean output functions_ will output 'homogeneous' `LONG` typed boolean values of `1` for `true` and `0` for `false`. Additionally,
+In Druid 28.0 and later, `druid.expressions.useStrictBooleans=true` is set by default. Logical operations treat `null` values as "unknown" for SQL compatible behavior. _All boolean output functions_ will output 'homogeneous' `LONG` typed boolean values of `1` for `true` and `0` for `false`.
For the "or" operator:
* `true || null`, `null || true`, -> `1`
@@ -322,15 +320,8 @@
* `false && null`, `null && false` -> `0`
Druid currently still retains implicit conversion of `LONG`, `DOUBLE`, and `STRING` types into boolean values in both modes:
-* `LONG` or `DOUBLE` - any value greater than 0 is considered `true`, else `false`
-* `STRING` - the value `'true'` (case insensitive) is considered `true`, everything else is `false`.
-
-Legacy behavior:
-* `100 && 11` -> `11`
-* `0.7 || 0.3` -> `0.3`
-* `100 && 0` -> `0`
-* `'troo' && 'true'` -> `'troo'`
-* `'troo' || 'true'` -> `'true'`
+* `LONG` or `DOUBLE`: any value greater than 0 is considered `true`, else `false`.
+* `STRING`: the value `'true'` (case insensitive) is considered `true`, everything else is `false`.
SQL compatible behavior:
* `100 && 11` -> `1`
@@ -339,4 +330,11 @@
* `'troo' && 'true'` -> `0`
* `'troo' || 'true'` -> `1`
+Prior to Druid 28.0.0, `druid.expressions.useStrictBooleans=false` was the default. In this mode, boolean function expressions have inconsistent handling of true and false values. The logical 'and' and 'or' operators behave in a manner that is incompatible with SQL, even if SQL compatible null handling mode (`druid.generic.useDefaultValueForNull=false`) is enabled. Logical operators also pass through their input values, similar to many scripting languages, and treat `null` as false, which results in some rather strange behavior. Other boolean operations, such as comparisons and equality, retain their input types (e.g. `DOUBLE` comparison produces `1.0` for true and `0.0` for false), while many other boolean functions strictly produce `LONG` typed values of `1` for true and `0` for false. This legacy mode can still be enabled by setting `druid.expressions.useStrictBooleans=false`.
+Legacy behavior:
+* `100 && 11` -> `11`
+* `0.7 || 0.3` -> `0.3`
+* `100 && 0` -> `0`
+* `'troo' && 'true'` -> `'troo'`
+* `'troo' || 'true'` -> `'true'`
diff --git a/docs/querying/sql-data-types.md b/docs/querying/sql-data-types.md
index 6fb5cc0..6f01369 100644
--- a/docs/querying/sql-data-types.md
+++ b/docs/querying/sql-data-types.md
@@ -153,13 +153,13 @@
## Boolean logic
The [`druid.expressions.useStrictBooleans`](../configuration/index.md#expression-processing-configurations)
-runtime property controls Druid's boolean logic mode. For the most SQL compliant behavior, set this to `true`.
+runtime property controls Druid's boolean logic mode. For the most SQL compliant behavior, set this to `true` (the default).
-When `druid.expressions.useStrictBooleans = false` (the default mode), Druid uses three-valued logic for
+When `druid.expressions.useStrictBooleans = true`, Druid uses three-valued logic for
[expressions](math-expr.md) evaluation, such as `expression` virtual columns or `expression` filters.
However, even in this mode, Druid uses two-valued logic for filter types other than `expression`.
-When `druid.expressions.useStrictBooleans = true` (legacy mode), Druid uses two-valued logic.
+When `druid.expressions.useStrictBooleans = false` (legacy mode), Druid uses two-valued logic.
## Nested columns
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java
index b510b6a..52fec4c 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerDiscoveryTest.java
@@ -58,71 +58,11 @@
private InputSourceSampler inputSourceSampler = new InputSourceSampler(OBJECT_MAPPER);
@Test
- public void testDiscoveredTypes()
+ public void testDiscoveredTypesNonStrictBooleans()
{
- final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
- final SamplerResponse response = inputSourceSampler.sample(
- inputSource,
- new JsonInputFormat(null, null, null, null, null),
- new DataSchema(
- "test",
- new TimestampSpec("t", null, null),
- DimensionsSpec.builder().useSchemaDiscovery(true).build(),
- null,
- null,
- null
- ),
- null
- );
- Assert.assertEquals(6, response.getNumRowsRead());
- Assert.assertEquals(5, response.getNumRowsIndexed());
- Assert.assertEquals(6, response.getData().size());
- Assert.assertEquals(
- ImmutableList.of(
- new StringDimensionSchema("string"),
- new LongDimensionSchema("long"),
- new DoubleDimensionSchema("double"),
- new StringDimensionSchema("bool"),
- new StringDimensionSchema("variant"),
- new AutoTypeColumnSchema("array"),
- new AutoTypeColumnSchema("nested")
- ),
- response.getLogicalDimensions()
- );
-
- Assert.assertEquals(
- ImmutableList.of(
- new AutoTypeColumnSchema("string"),
- new AutoTypeColumnSchema("long"),
- new AutoTypeColumnSchema("double"),
- new AutoTypeColumnSchema("bool"),
- new AutoTypeColumnSchema("variant"),
- new AutoTypeColumnSchema("array"),
- new AutoTypeColumnSchema("nested")
- ),
- response.getPhysicalDimensions()
- );
- Assert.assertEquals(
- RowSignature.builder()
- .addTimeColumn()
- .add("string", ColumnType.STRING)
- .add("long", ColumnType.LONG)
- .add("double", ColumnType.DOUBLE)
- .add("bool", ColumnType.STRING)
- .add("variant", ColumnType.STRING)
- .add("array", ColumnType.LONG_ARRAY)
- .add("nested", ColumnType.NESTED_DATA)
- .build(),
- response.getLogicalSegmentSchema()
- );
- }
-
- @Test
- public void testDiscoveredTypesStrictBooleans()
- {
try {
- ExpressionProcessing.initializeForStrictBooleansTests(true);
+ ExpressionProcessing.initializeForStrictBooleansTests(false);
final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
final SamplerResponse response = inputSourceSampler.sample(
inputSource,
@@ -146,7 +86,7 @@
new StringDimensionSchema("string"),
new LongDimensionSchema("long"),
new DoubleDimensionSchema("double"),
- new LongDimensionSchema("bool"),
+ new StringDimensionSchema("bool"),
new StringDimensionSchema("variant"),
new AutoTypeColumnSchema("array"),
new AutoTypeColumnSchema("nested")
@@ -172,7 +112,7 @@
.add("string", ColumnType.STRING)
.add("long", ColumnType.LONG)
.add("double", ColumnType.DOUBLE)
- .add("bool", ColumnType.LONG)
+ .add("bool", ColumnType.STRING)
.add("variant", ColumnType.STRING)
.add("array", ColumnType.LONG_ARRAY)
.add("nested", ColumnType.NESTED_DATA)
@@ -186,6 +126,67 @@
}
@Test
+ public void testDiscoveredTypesStrictBooleans()
+ {
+ final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
+ final SamplerResponse response = inputSourceSampler.sample(
+ inputSource,
+ new JsonInputFormat(null, null, null, null, null),
+ new DataSchema(
+ "test",
+ new TimestampSpec("t", null, null),
+ DimensionsSpec.builder().useSchemaDiscovery(true).build(),
+ null,
+ null,
+ null
+ ),
+ null
+ );
+
+ Assert.assertEquals(6, response.getNumRowsRead());
+ Assert.assertEquals(5, response.getNumRowsIndexed());
+ Assert.assertEquals(6, response.getData().size());
+ Assert.assertEquals(
+ ImmutableList.of(
+ new StringDimensionSchema("string"),
+ new LongDimensionSchema("long"),
+ new DoubleDimensionSchema("double"),
+ new LongDimensionSchema("bool"),
+ new StringDimensionSchema("variant"),
+ new AutoTypeColumnSchema("array"),
+ new AutoTypeColumnSchema("nested")
+ ),
+ response.getLogicalDimensions()
+ );
+
+ Assert.assertEquals(
+ ImmutableList.of(
+ new AutoTypeColumnSchema("string"),
+ new AutoTypeColumnSchema("long"),
+ new AutoTypeColumnSchema("double"),
+ new AutoTypeColumnSchema("bool"),
+ new AutoTypeColumnSchema("variant"),
+ new AutoTypeColumnSchema("array"),
+ new AutoTypeColumnSchema("nested")
+ ),
+ response.getPhysicalDimensions()
+ );
+ Assert.assertEquals(
+ RowSignature.builder()
+ .addTimeColumn()
+ .add("string", ColumnType.STRING)
+ .add("long", ColumnType.LONG)
+ .add("double", ColumnType.DOUBLE)
+ .add("bool", ColumnType.LONG)
+ .add("variant", ColumnType.STRING)
+ .add("array", ColumnType.LONG_ARRAY)
+ .add("nested", ColumnType.NESTED_DATA)
+ .build(),
+ response.getLogicalSegmentSchema()
+ );
+ }
+
+ @Test
public void testTypesClassicDiscovery()
{
final InputSource inputSource = new InlineInputSource(Strings.join(STR_JSON_ROWS, '\n'));
diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
index 4bd29a1..af1faf9 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java
@@ -561,9 +561,6 @@
return ofDouble((Number) value);
}
if (value instanceof Boolean) {
- if (ExpressionProcessing.useStrictBooleans()) {
- return ofLongBoolean((Boolean) value);
- }
return ofDouble(Evals.asDouble((Boolean) value));
}
if (value instanceof String) {
diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
index 44e3b94..2f12bd3 100644
--- a/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
+++ b/processing/src/main/java/org/apache/druid/math/expr/ExpressionProcessingConfig.java
@@ -50,7 +50,11 @@
@JsonProperty("homogenizeNullMultiValueStringArrays") @Nullable Boolean homogenizeNullMultiValueStringArrays
)
{
- this.useStrictBooleans = getWithPropertyFallbackFalse(useStrictBooleans, NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING);
+ this.useStrictBooleans = getWithPropertyFallback(
+ useStrictBooleans,
+ NULL_HANDLING_LEGACY_LOGICAL_OPS_STRING,
+ "true"
+ );
this.processArraysAsMultiValueStrings = getWithPropertyFallbackFalse(
processArraysAsMultiValueStrings,
PROCESS_ARRAYS_AS_MULTIVALUE_STRINGS_CONFIG_STRING
@@ -78,6 +82,11 @@
private static boolean getWithPropertyFallbackFalse(@Nullable Boolean value, String property)
{
- return value != null ? value : Boolean.valueOf(System.getProperty(property, "false"));
+ return getWithPropertyFallback(value, property, "false");
+ }
+
+ private static boolean getWithPropertyFallback(@Nullable Boolean value, String property, String fallback)
+ {
+ return value != null ? value : Boolean.valueOf(System.getProperty(property, fallback));
}
}
diff --git a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
index 16ecc39..97a91cb 100644
--- a/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
+++ b/processing/src/test/java/org/apache/druid/math/expr/EvalTest.java
@@ -1382,9 +1382,6 @@
// strings
assertBestEffortOf("stringy", ExpressionType.STRING, "stringy");
- // by default, booleans are handled as strings
- assertBestEffortOf(true, ExpressionType.STRING, "true");
- assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"});
assertBestEffortOf(
new byte[]{1, 2, 3, 4},
@@ -1396,11 +1393,15 @@
assertBestEffortOf(1L, ExpressionType.LONG, 1L);
assertBestEffortOf(1, ExpressionType.LONG, 1L);
+ // by default, booleans are handled as longs
+ assertBestEffortOf(true, ExpressionType.LONG, 1L);
+ assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L});
+
try {
- // in strict boolean mode, they are longs
- ExpressionProcessing.initializeForStrictBooleansTests(true);
- assertBestEffortOf(true, ExpressionType.LONG, 1L);
- assertBestEffortOf(Arrays.asList(true, false), ExpressionType.LONG_ARRAY, new Object[]{1L, 0L});
+ // in non-strict boolean mode, they are strings
+ ExpressionProcessing.initializeForStrictBooleansTests(false);
+ assertBestEffortOf(true, ExpressionType.STRING, "true");
+ assertBestEffortOf(Arrays.asList(true, false), ExpressionType.STRING_ARRAY, new Object[]{"true", "false"});
}
finally {
// reset
diff --git a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
index ad4dccd..a032e99 100644
--- a/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
+++ b/processing/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java
@@ -26,9 +26,7 @@
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.vector.ExprEvalVector;
import org.apache.druid.testing.InitializedNullHandlingTest;
-import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.BeforeClass;
import org.junit.Test;
import javax.annotation.Nullable;
@@ -64,18 +62,6 @@
.put("boolString2", ExpressionType.STRING)
.build();
- @BeforeClass
- public static void setupTests()
- {
- ExpressionProcessing.initializeForStrictBooleansTests(true);
- }
-
- @AfterClass
- public static void teardownTests()
- {
- ExpressionProcessing.initializeForTests();
- }
-
@Test
public void testUnaryOperators()
{
diff --git a/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java b/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java
index 71e9779..ba23a54 100644
--- a/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java
+++ b/processing/src/test/java/org/apache/druid/query/expression/VectorExpressionsSanityTest.java
@@ -26,13 +26,10 @@
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.ExprEval;
-import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.math.expr.ExpressionType;
import org.apache.druid.math.expr.VectorExprSanityTest;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.joda.time.DateTime;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Test;
import java.util.Map;
@@ -62,18 +59,6 @@
.put("boolString2", ExpressionType.STRING)
.build();
- @BeforeClass
- public static void setupTests()
- {
- ExpressionProcessing.initializeForStrictBooleansTests(true);
- }
-
- @AfterClass
- public static void teardownTests()
- {
- ExpressionProcessing.initializeForTests();
- }
-
static void testExpression(String expr, Expr parsed, Map<String, ExpressionType> types)
{
log.debug("[%s]", expr);
diff --git a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
index b4fe9ce..e06b5b5 100644
--- a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
+++ b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java
@@ -600,7 +600,77 @@
}
@Test
- public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypes() throws Exception
+ public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesNonStrictBooleans() throws Exception
+ {
+
+ try {
+ ExpressionProcessing.initializeForStrictBooleansTests(false);
+ Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
+ .dataSource("test_datasource")
+ .intervals(
+ new MultipleIntervalSegmentSpec(
+ Collections.singletonList(Intervals.ETERNITY)
+ )
+ )
+ .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+ .limit(100)
+ .context(ImmutableMap.of());
+ Query<ScanResultValue> scanQuery = builder.build();
+ final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")};
+ List<Segment> realtimeSegs = ImmutableList.of(
+ NestedDataTestUtils.createIncrementalIndex(
+ tempFolder,
+ NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
+ NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
+ NestedDataTestUtils.TIMESTAMP_SPEC,
+ NestedDataTestUtils.AUTO_DISCOVERY,
+ TransformSpec.NONE,
+ aggs,
+ Granularities.NONE,
+ true
+ )
+ );
+ List<Segment> segs = NestedDataTestUtils.createSegments(
+ tempFolder,
+ closer,
+ NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
+ NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
+ NestedDataTestUtils.TIMESTAMP_SPEC,
+ NestedDataTestUtils.AUTO_DISCOVERY,
+ TransformSpec.NONE,
+ aggs,
+ Granularities.NONE,
+ true,
+ IndexSpec.DEFAULT
+ );
+
+
+ final Sequence<ScanResultValue> seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery);
+ final Sequence<ScanResultValue> seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery);
+
+ List<ScanResultValue> resultsRealtime = seq.toList();
+ List<ScanResultValue> resultsSegments = seq2.toList();
+ logResults(resultsSegments);
+ logResults(resultsRealtime);
+ Assert.assertEquals(1, resultsRealtime.size());
+ Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
+ Assert.assertEquals(
+ "["
+ + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
+ resultsSegments.get(0).getEvents().toString()
+ );
+ Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
+ }
+ finally {
+ ExpressionProcessing.initializeForTests();
+ }
+ }
+
+ @Test
+ public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception
{
Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
.dataSource("test_datasource")
@@ -653,85 +723,16 @@
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
Assert.assertEquals(
"["
- + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
- + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
- + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
- + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [true, true], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
+ + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
+ + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
resultsSegments.get(0).getEvents().toString()
);
Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
}
@Test
- public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryMoreArrayTypesStrictBooleans() throws Exception
- {
- try {
- ExpressionProcessing.initializeForStrictBooleansTests(true);
- Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
- .dataSource("test_datasource")
- .intervals(
- new MultipleIntervalSegmentSpec(
- Collections.singletonList(Intervals.ETERNITY)
- )
- )
- .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
- .limit(100)
- .context(ImmutableMap.of());
- Query<ScanResultValue> scanQuery = builder.build();
- final AggregatorFactory[] aggs = new AggregatorFactory[]{new CountAggregatorFactory("count")};
- List<Segment> realtimeSegs = ImmutableList.of(
- NestedDataTestUtils.createIncrementalIndex(
- tempFolder,
- NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
- NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
- NestedDataTestUtils.TIMESTAMP_SPEC,
- NestedDataTestUtils.AUTO_DISCOVERY,
- TransformSpec.NONE,
- aggs,
- Granularities.NONE,
- true
- )
- );
- List<Segment> segs = NestedDataTestUtils.createSegments(
- tempFolder,
- closer,
- NestedDataTestUtils.ARRAY_TYPES_DATA_FILE_2,
- NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT,
- NestedDataTestUtils.TIMESTAMP_SPEC,
- NestedDataTestUtils.AUTO_DISCOVERY,
- TransformSpec.NONE,
- aggs,
- Granularities.NONE,
- true,
- IndexSpec.DEFAULT
- );
-
-
- final Sequence<ScanResultValue> seq = helper.runQueryOnSegmentsObjs(realtimeSegs, scanQuery);
- final Sequence<ScanResultValue> seq2 = helper.runQueryOnSegmentsObjs(segs, scanQuery);
-
- List<ScanResultValue> resultsRealtime = seq.toList();
- List<ScanResultValue> resultsSegments = seq2.toList();
- logResults(resultsSegments);
- logResults(resultsRealtime);
- Assert.assertEquals(1, resultsRealtime.size());
- Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
- Assert.assertEquals(
- "["
- + "[978652800000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
- + "[978739200000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
- + "[978825600000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1], "
- + "[978912000000, [A, A], [null, null], [1, 1], [0.1, 0.1], [1, 1], [null, null], {s_str1=[A, A], s_str2=[null, null], s_num_int=[1, 1], s_num_float=[0.1, 0.1], s_bool=[true, true], s_null=[null, null]}, 1]]",
- resultsSegments.get(0).getEvents().toString()
- );
- Assert.assertEquals(resultsSegments.get(0).getEvents().toString(), resultsRealtime.get(0).getEvents().toString());
- }
- finally {
- ExpressionProcessing.initializeForTests();
- }
- }
-
- @Test
public void testIngestAndScanSegmentsRealtimeSchemaDiscoveryTypeGauntlet() throws Exception
{
Druids.ScanQueryBuilder builder = Druids.newScanQueryBuilder()
@@ -785,12 +786,12 @@
Assert.assertEquals(resultsRealtime.size(), resultsSegments.size());
if (NullHandling.replaceWithDefault()) {
Assert.assertEquals(
- "[[1672531200000, null, 0, 0.0, true, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 2, 0.0, false, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, true, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, true, 1, 0.0, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, 0, 4.4, true, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, false, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [true, null, true], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
+ "[[1672531200000, null, 0, 0.0, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 2, 0.0, 0, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [1, 1], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, 1, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [1, 0, 1], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, 1, 1, 0.0, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, 0, 1], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, 0, 4.4, 1, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [0], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, 0, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, 0, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [1, null, 1], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
resultsSegments.get(0).getEvents().toString()
);
} else {
Assert.assertEquals(
- "[[1672531200000, null, null, null, true, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [true, false, true], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, false, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [true, true], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, true, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [true, false, true], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, true, 1, null, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, false, true], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, null, 4.4, true, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [false], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, false, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [true, null, true], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
+ "[[1672531200000, null, null, null, 1, 51, -0.13, 1, [], [51, -35], {a=700, b={x=g, y=1.1, z=[9, null, 9, 9]}}, {x=400, y=[{l=[null], m=100, n=5}, {l=[a, b, c], m=a, n=1}], z={}}, null, [a, b], null, [2, 3], null, [null], null, [1, 0, 1], null, [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, , 2, null, 0, b, 1.1, b, 2, b, {a=200, b={x=b, y=1.1, z=[2, 4, 6]}}, {x=10, y=[{l=[b, b, c], m=b, n=2}, [1, 2, 3]], z={a=[5.5], b=false}}, [a, b, c], [null, b], [2, 3], null, [3.3, 4.4, 5.5], [999.0, null, 5.5], [null, null, 2.2], [1, 1], [null, [null], []], [{x=3}, {x=4}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, a, 1, 1.0, 1, 1, 1, 1, 1, 1, {a=100, b={x=a, y=1.1, z=[1, 2, 3, 4]}}, {x=1234, y=[{l=[a, b, c], m=a, n=1}, {l=[a, b, c], m=a, n=1}], z={a=[1.1, 2.2, 3.3], b=true}}, [a, b], [a, b], [1, 2, 3], [1, null, 3], [1.1, 2.2, 3.3], [1.1, 2.2, null], [a, 1, 2.2], [1, 0, 1], [[1, 2, null], [3, 4]], [{x=1}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, b, 4, 3.3, 1, 1, null, {}, 4, 1, {a=400, b={x=d, y=1.1, z=[3, 4]}}, {x=1234, z={a=[1.1, 2.2, 3.3], b=true}}, [d, e], [b, b], [1, 4], [1], [2.2, 3.3, 4.0], null, [a, b, c], [null, 0, 1], [[1, 2], [3, 4], [5, 6, 7]], [{x=null}, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, c, null, 4.4, 1, hello, -1000, {}, [], hello, {a=500, b={x=e, z=[1, 2, 3, 4]}}, {x=11, y=[], z={a=[null], b=false}}, null, null, [1, 2, 3], [], [1.1, 2.2, 3.3], null, null, [0], null, [{x=1000}, {y=2000}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, d, 5, 5.9, 0, null, 3.33, a, 6, null, {a=600, b={x=f, y=1.1, z=[6, 7, 8, 9]}}, null, [a, b], null, null, [null, 2, 9], null, [999.0, 5.5, null], [a, 1, 2.2], [], [[1], [1, 2, null]], [{a=1}, {b=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1], [1672531200000, null, 3, 2.0, null, 3.0, 1.0, 3.3, 3, 3.0, {a=300}, {x=4.4, y=[{l=[], m=100, n=3}, {l=[a]}, {l=[b], n=[]}], z={a=[], b=true}}, [b, c], [d, null, b], [1, 2, 3, 4], [1, 2, 3], [1.1, 3.3], [null, 2.2, null], [1, null, 1], [1, null, 1], [[1], null, [1, 2, 3]], [null, {x=2}], null, hello, 1234, 1.234, {x=1, y=hello, z={a=1.1, b=1234, c=[a, b, c]}}, [a, b, c], [1, 2, 3], [1.1, 2.2, 3.3], [], {}, [null, null], [{}, {}, {}], [{a=b, x=1, y=1.3}], 1]]",
resultsSegments.get(0).getEvents().toString()
);
}
diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterStrictBooleansTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java
similarity index 88%
rename from processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterStrictBooleansTest.java
rename to processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java
index 589a735..3716f6c 100644
--- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterStrictBooleansTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterNonStrictBooleansTest.java
@@ -31,9 +31,9 @@
import java.io.Closeable;
@RunWith(Parameterized.class)
-public class ExpressionFilterStrictBooleansTest extends ExpressionFilterTest
+public class ExpressionFilterNonStrictBooleansTest extends ExpressionFilterTest
{
- public ExpressionFilterStrictBooleansTest(
+ public ExpressionFilterNonStrictBooleansTest(
String testName,
IndexBuilder indexBuilder,
Function<IndexBuilder, Pair<StorageAdapter, Closeable>> finisher,
@@ -48,6 +48,6 @@
@Override
public void setup()
{
- ExpressionProcessing.initializeForStrictBooleansTests(true);
+ ExpressionProcessing.initializeForStrictBooleansTests(false);
}
}
diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
index ac0ef94..9c40ccb 100644
--- a/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/filter/ExpressionFilterTest.java
@@ -122,7 +122,7 @@
@Before
public void setup()
{
- ExpressionProcessing.initializeForStrictBooleansTests(false);
+ ExpressionProcessing.initializeForStrictBooleansTests(true);
}
@After
diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java
index 9171c07..2517ce1 100644
--- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexTest.java
@@ -400,7 +400,7 @@
Assert.assertEquals(ColumnType.STRING, index.getColumnCapabilities("bool_string").toColumnType());
Assert.assertEquals(ColumnType.LONG, index.getColumnCapabilities("bool_long").toColumnType());
// depends on value of 'druid.expressions.useStrictBooleans', current default is false which parses as strings
- Assert.assertEquals(ColumnType.STRING, index.getColumnCapabilities("bool_auto").toColumnType());
+ Assert.assertEquals(ColumnType.LONG, index.getColumnCapabilities("bool_auto").toColumnType());
Assert.assertEquals(ColumnType.STRING_ARRAY, index.getColumnCapabilities("array_string").toColumnType());
Assert.assertEquals(ColumnType.LONG_ARRAY, index.getColumnCapabilities("array_long").toColumnType());
Assert.assertEquals(ColumnType.DOUBLE_ARRAY, index.getColumnCapabilities("array_double").toColumnType());
diff --git a/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java b/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java
index 4e1fad1..972932e 100644
--- a/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/transform/TransformerTest.java
@@ -155,7 +155,8 @@
final InputRow actual = transformer.transform(row);
Assert.assertNotNull(actual);
Assert.assertEquals(ImmutableList.of("dim"), actual.getDimensions());
- Assert.assertEquals(5L, actual.getRaw("dim"));
+ // booleans are longs by default, so strlen of false (0L) is 1
+ Assert.assertEquals(1L, actual.getRaw("dim"));
Assert.assertEquals(row.getTimestamp(), actual.getTimestamp());
}
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
index bb3dda4..406f94b 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java
@@ -5661,7 +5661,7 @@
"",
0L,
0.0D,
- "true",
+ 1L,
"51",
-0.13D,
"1",
@@ -5676,7 +5676,7 @@
null,
"[null]",
null,
- "[\"true\",\"false\",\"true\"]",
+ "[1,0,1]",
null,
"[{\"x\":1},{\"x\":2}]",
"",
@@ -5699,7 +5699,7 @@
"",
2L,
0.0D,
- "false",
+ 0L,
"b",
1.1D,
"\"b\"",
@@ -5714,7 +5714,7 @@
"[3.3,4.4,5.5]",
"[999.0,null,5.5]",
"[null,null,2.2]",
- "[\"true\",\"true\"]",
+ "[1,1]",
"[null,[null],[]]",
"[{\"x\":3},{\"x\":4}]",
"",
@@ -5737,7 +5737,7 @@
"a",
1L,
1.0D,
- "true",
+ 1L,
"1",
1.0D,
"1",
@@ -5752,7 +5752,7 @@
"[1.1,2.2,3.3]",
"[1.1,2.2,null]",
"[\"a\",\"1\",\"2.2\"]",
- "[\"true\",\"false\",\"true\"]",
+ "[1,0,1]",
"[[1,2,null],[3,4]]",
"[{\"x\":1},{\"x\":2}]",
"",
@@ -5775,7 +5775,7 @@
"b",
4L,
3.3D,
- "true",
+ 1L,
"1",
0.0D,
"{}",
@@ -5790,7 +5790,7 @@
"[2.2,3.3,4.0]",
null,
"[\"a\",\"b\",\"c\"]",
- "[null,\"false\",\"true\"]",
+ "[null,0,1]",
"[[1,2],[3,4],[5,6,7]]",
"[{\"x\":null},{\"x\":2}]",
"",
@@ -5813,7 +5813,7 @@
"c",
0L,
4.4D,
- "true",
+ 1L,
"hello",
-1000.0D,
"{}",
@@ -5828,7 +5828,7 @@
"[1.1,2.2,3.3]",
null,
null,
- "[\"false\"]",
+ "[0]",
null,
"[{\"x\":1000},{\"y\":2000}]",
"",
@@ -5851,7 +5851,7 @@
"d",
5L,
5.9D,
- "false",
+ 0L,
"",
3.33D,
"\"a\"",
@@ -5889,7 +5889,7 @@
"null",
3L,
2.0D,
- "",
+ 0L,
"3.0",
1.0D,
"3.3",
@@ -5904,7 +5904,7 @@
"[1.1,3.3]",
"[null,2.2,null]",
"[1,null,1]",
- "[\"true\",null,\"true\"]",
+ "[1,null,1]",
"[[1],null,[1,2,3]]",
"[null,{\"x\":2}]",
"",
@@ -5929,7 +5929,7 @@
null,
null,
null,
- "true",
+ 1L,
"51",
-0.13D,
"1",
@@ -5944,7 +5944,7 @@
null,
"[null]",
null,
- "[\"true\",\"false\",\"true\"]",
+ "[1,0,1]",
null,
"[{\"x\":1},{\"x\":2}]",
null,
@@ -5967,7 +5967,7 @@
"",
2L,
null,
- "false",
+ 0L,
"b",
1.1D,
"\"b\"",
@@ -5982,7 +5982,7 @@
"[3.3,4.4,5.5]",
"[999.0,null,5.5]",
"[null,null,2.2]",
- "[\"true\",\"true\"]",
+ "[1,1]",
"[null,[null],[]]",
"[{\"x\":3},{\"x\":4}]",
null,
@@ -6005,7 +6005,7 @@
"a",
1L,
1.0D,
- "true",
+ 1L,
"1",
1.0D,
"1",
@@ -6020,7 +6020,7 @@
"[1.1,2.2,3.3]",
"[1.1,2.2,null]",
"[\"a\",\"1\",\"2.2\"]",
- "[\"true\",\"false\",\"true\"]",
+ "[1,0,1]",
"[[1,2,null],[3,4]]",
"[{\"x\":1},{\"x\":2}]",
null,
@@ -6043,7 +6043,7 @@
"b",
4L,
3.3D,
- "true",
+ 1L,
"1",
null,
"{}",
@@ -6058,7 +6058,7 @@
"[2.2,3.3,4.0]",
null,
"[\"a\",\"b\",\"c\"]",
- "[null,\"false\",\"true\"]",
+ "[null,0,1]",
"[[1,2],[3,4],[5,6,7]]",
"[{\"x\":null},{\"x\":2}]",
null,
@@ -6081,7 +6081,7 @@
"c",
null,
4.4D,
- "true",
+ 1L,
"hello",
-1000.0D,
"{}",
@@ -6096,7 +6096,7 @@
"[1.1,2.2,3.3]",
null,
null,
- "[\"false\"]",
+ "[0]",
null,
"[{\"x\":1000},{\"y\":2000}]",
null,
@@ -6119,7 +6119,7 @@
"d",
5L,
5.9D,
- "false",
+ 0L,
null,
3.33D,
"\"a\"",
@@ -6172,7 +6172,7 @@
"[1.1,3.3]",
"[null,2.2,null]",
"[1,null,1]",
- "[\"true\",null,\"true\"]",
+ "[1,null,1]",
"[[1],null,[1,2,3]]",
"[null,{\"x\":2}]",
null,
@@ -6196,7 +6196,7 @@
.add("str", ColumnType.STRING)
.add("long", ColumnType.LONG)
.add("double", ColumnType.DOUBLE)
- .add("bool", ColumnType.STRING)
+ .add("bool", ColumnType.LONG)
.add("variant", ColumnType.STRING)
.add("variantNumeric", ColumnType.DOUBLE)
.add("variantEmptyObj", ColumnType.NESTED_DATA)
@@ -6211,7 +6211,7 @@
.add("arrayDouble", ColumnType.DOUBLE_ARRAY)
.add("arrayDoubleNulls", ColumnType.DOUBLE_ARRAY)
.add("arrayVariant", ColumnType.STRING_ARRAY)
- .add("arrayBool", ColumnType.STRING_ARRAY)
+ .add("arrayBool", ColumnType.LONG_ARRAY)
.add("arrayNestedLong", ColumnType.NESTED_DATA)
.add("arrayObject", ColumnType.NESTED_DATA)
.add("null", ColumnType.STRING)
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java
index 157a9e2..2ac0574 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java
@@ -29,7 +29,6 @@
import org.apache.druid.java.util.common.guava.Yielders;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.math.expr.ExpressionProcessing;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.segment.QueryableIndex;
@@ -113,7 +112,6 @@
@BeforeClass
public static void setupClass()
{
- ExpressionProcessing.initializeForStrictBooleansTests(true);
CLOSER = Closer.create();
final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
@@ -162,7 +160,6 @@
public static void teardownClass() throws IOException
{
CLOSER.close();
- ExpressionProcessing.initializeForTests();
}
@Parameterized.Parameters(name = "query = {0}")
