| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| package org.apache.parquet.schema; |
| |
| import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; |
| import org.apache.parquet.schema.Type.Repetition; |
| import org.junit.Assert; |
| import org.junit.Test; |
| |
| import java.util.concurrent.Callable; |
| |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MICROS; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.MILLIS; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit.NANOS; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.bsonType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.dateType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.decimalType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.intType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.jsonType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.stringType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.timeType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.timestampType; |
| import static org.apache.parquet.schema.LogicalTypeAnnotation.uuidType; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.DOUBLE; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FLOAT; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; |
| import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT96; |
| import static org.apache.parquet.schema.Type.Repetition.REQUIRED; |
| import static org.junit.Assert.assertEquals; |
| |
| public class TestTypeBuildersWithLogicalTypes { |
| @Test |
| public void testGroupTypeConstruction() { |
| PrimitiveType f1 = Types.required(BINARY).as(stringType()).named("f1"); |
| PrimitiveType f2 = Types.required(INT32).named("f2"); |
| PrimitiveType f3 = Types.optional(INT32).named("f3"); |
| String name = "group"; |
| for (Repetition repetition : Repetition.values()) { |
| GroupType expected = new GroupType(repetition, name, |
| f1, |
| new GroupType(repetition, "g1", f2, f3)); |
| GroupType built = Types.buildGroup(repetition) |
| .addField(f1) |
| .group(repetition).addFields(f2, f3).named("g1") |
| .named(name); |
| Assert.assertEquals(expected, built); |
| |
| switch (repetition) { |
| case REQUIRED: |
| built = Types.requiredGroup() |
| .addField(f1) |
| .requiredGroup().addFields(f2, f3).named("g1") |
| .named(name); |
| break; |
| case OPTIONAL: |
| built = Types.optionalGroup() |
| .addField(f1) |
| .optionalGroup().addFields(f2, f3).named("g1") |
| .named(name); |
| break; |
| case REPEATED: |
| built = Types.repeatedGroup() |
| .addField(f1) |
| .repeatedGroup().addFields(f2, f3).named("g1") |
| .named(name); |
| break; |
| } |
| Assert.assertEquals(expected, built); |
| } |
| } |
| |
| @Test |
| public void testDecimalAnnotation() { |
| // int32 primitive type |
| MessageType expected = new MessageType("DecimalMessage", |
| new PrimitiveType(REQUIRED, INT32, 0, "aDecimal", |
| decimalType(2, 9), null)); |
| MessageType builderType = Types.buildMessage() |
| .required(INT32) |
| .as(decimalType(2, 9)) |
| .named("aDecimal") |
| .named("DecimalMessage"); |
| Assert.assertEquals(expected, builderType); |
| // int64 primitive type |
| expected = new MessageType("DecimalMessage", |
| new PrimitiveType(REQUIRED, INT64, 0, "aDecimal", |
| decimalType(2, 18), null)); |
| builderType = Types.buildMessage() |
| .required(INT64) |
| .as(decimalType(2, 18)).precision(18).scale(2) |
| .named("aDecimal") |
| .named("DecimalMessage"); |
| Assert.assertEquals(expected, builderType); |
| // binary primitive type |
| expected = new MessageType("DecimalMessage", |
| new PrimitiveType(REQUIRED, BINARY, 0, "aDecimal", |
| decimalType(2, 9), null)); |
| builderType = Types.buildMessage() |
| .required(BINARY).as(decimalType(2, 9)) |
| .named("aDecimal") |
| .named("DecimalMessage"); |
| Assert.assertEquals(expected, builderType); |
| // fixed primitive type |
| expected = new MessageType("DecimalMessage", |
| new PrimitiveType(REQUIRED, FIXED_LEN_BYTE_ARRAY, 4, "aDecimal", |
| decimalType(2, 9), null)); |
| builderType = Types.buildMessage() |
| .required(FIXED_LEN_BYTE_ARRAY).length(4) |
| .as(decimalType(2, 9)) |
| .named("aDecimal") |
| .named("DecimalMessage"); |
| Assert.assertEquals(expected, builderType); |
| } |
| |
| @Test |
| public void testDecimalAnnotationPrecisionScaleBound() { |
| assertThrows("Should reject scale greater than precision", |
| IllegalArgumentException.class, () -> Types.buildMessage() |
| .required(INT32).as(decimalType(4, 3)) |
| .named("aDecimal") |
| .named("DecimalMessage")); |
| assertThrows("Should reject scale greater than precision", |
| IllegalArgumentException.class, () -> Types.buildMessage() |
| .required(INT64).as(decimalType(4, 3)) |
| .named("aDecimal") |
| .named("DecimalMessage")); |
| assertThrows("Should reject scale greater than precision", |
| IllegalArgumentException.class, () -> Types.buildMessage() |
| .required(BINARY).as(decimalType(4, 3)) |
| .named("aDecimal") |
| .named("DecimalMessage")); |
| assertThrows("Should reject scale greater than precision", |
| IllegalArgumentException.class, () -> Types.buildMessage() |
| .required(FIXED_LEN_BYTE_ARRAY).length(7) |
| .as(decimalType(4, 3)) |
| .named("aDecimal") |
| .named("DecimalMessage") |
| ); |
| } |
| |
| @Test |
| public void testDecimalAnnotationLengthCheck() { |
| // maximum precision for 4 bytes is 9 |
| assertThrows("should reject precision 10 with length 4", |
| IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(4) |
| .as(decimalType(2, 10)) |
| .named("aDecimal")); |
| assertThrows("should reject precision 10 with length 4", |
| IllegalStateException.class, () -> Types.required(INT32) |
| .as(decimalType(2, 10)) |
| .named("aDecimal")); |
| // maximum precision for 8 bytes is 19 |
| assertThrows("should reject precision 19 with length 8", |
| IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(8) |
| .as(decimalType(4, 19)) |
| .named("aDecimal")); |
| assertThrows("should reject precision 19 with length 8", |
| IllegalStateException.class, () -> Types.required(INT64).length(8) |
| .as(decimalType(4, 19)) |
| .named("aDecimal") |
| ); |
| } |
| |
| @Test |
| public void testDECIMALAnnotationRejectsUnsupportedTypes() { |
| PrimitiveTypeName[] unsupported = new PrimitiveTypeName[]{ |
| BOOLEAN, INT96, DOUBLE, FLOAT |
| }; |
| for (final PrimitiveTypeName type : unsupported) { |
| assertThrows("Should reject non-binary type: " + type, |
| IllegalStateException.class, () -> Types.required(type) |
| .as(decimalType(2, 9)) |
| .named("d")); |
| } |
| } |
| |
| @Test |
| public void testBinaryAnnotations() { |
| LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] { |
| stringType(), jsonType(), bsonType()}; |
| for (final LogicalTypeAnnotation logicalType : types) { |
| PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "col", logicalType); |
| PrimitiveType string = Types.required(BINARY).as(logicalType).named("col"); |
| Assert.assertEquals(expected, string); |
| } |
| } |
| |
| @Test |
| public void testBinaryAnnotationsRejectsNonBinary() { |
| LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] { |
| stringType(), jsonType(), bsonType()}; |
| for (final LogicalTypeAnnotation logicalType : types) { |
| PrimitiveTypeName[] nonBinary = new PrimitiveTypeName[]{ |
| BOOLEAN, INT32, INT64, INT96, DOUBLE, FLOAT |
| }; |
| for (final PrimitiveTypeName type : nonBinary) { |
| assertThrows("Should reject non-binary type: " + type, |
| IllegalStateException.class, () -> Types.required(type).as(logicalType).named("col")); |
| } |
| assertThrows("Should reject non-binary type: FIXED_LEN_BYTE_ARRAY", |
| IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1) |
| .as(logicalType).named("col")); |
| } |
| } |
| |
| @Test |
| public void testInt32Annotations() { |
| LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] { |
| dateType(), timeType(true, MILLIS), timeType(false, MILLIS), |
| intType(8, false), intType(16, false), intType(32, false), |
| intType(8, true), intType(16, true), intType(32, true)}; |
| for (LogicalTypeAnnotation logicalType : types) { |
| PrimitiveType expected = new PrimitiveType(REQUIRED, INT32, "col", logicalType); |
| PrimitiveType date = Types.required(INT32).as(logicalType).named("col"); |
| Assert.assertEquals(expected, date); |
| } |
| } |
| |
| @Test |
| public void testInt32AnnotationsRejectNonInt32() { |
| LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] { |
| dateType(), timeType(true, MILLIS), timeType(false, MILLIS), |
| intType(8, false), intType(16, false), intType(32, false), |
| intType(8, true), intType(16, true), intType(32, true)}; |
| for (final LogicalTypeAnnotation logicalType : types) { |
| PrimitiveTypeName[] nonInt32 = new PrimitiveTypeName[]{ |
| BOOLEAN, INT64, INT96, DOUBLE, FLOAT, BINARY |
| }; |
| for (final PrimitiveTypeName type : nonInt32) { |
| assertThrows("Should reject non-int32 type: " + type, |
| IllegalStateException.class, () -> Types.required(type).as(logicalType).named("col")); |
| } |
| assertThrows("Should reject non-int32 type: FIXED_LEN_BYTE_ARRAY", |
| IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1) |
| .as(logicalType).named("col")); |
| } |
| } |
| |
| @Test |
| public void testInt64Annotations() { |
| LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] { |
| timeType(true, MICROS), timeType(false, MICROS), |
| timeType(true, NANOS), timeType(false, NANOS), |
| timestampType(true, MILLIS), timestampType(false, MILLIS), |
| timestampType(true, MICROS), timestampType(false, MICROS), |
| timestampType(true, NANOS), timestampType(false, NANOS), |
| intType(64, true), intType(64, false)}; |
| for (LogicalTypeAnnotation logicalType : types) { |
| PrimitiveType expected = new PrimitiveType(REQUIRED, INT64, "col", logicalType); |
| PrimitiveType date = Types.required(INT64).as(logicalType).named("col"); |
| Assert.assertEquals(expected, date); |
| } |
| } |
| |
| @Test |
| public void testInt64AnnotationsRejectNonInt64() { |
| LogicalTypeAnnotation[] types = new LogicalTypeAnnotation[] { |
| timeType(true, MICROS), timeType(false, MICROS), |
| timeType(true, NANOS), timeType(false, NANOS), |
| timestampType(true, MILLIS), timestampType(false, MILLIS), |
| timestampType(true, MICROS), timestampType(false, MICROS), |
| timestampType(true, NANOS), timestampType(false, NANOS), |
| intType(64, true), intType(64, false)}; |
| for (final LogicalTypeAnnotation logicalType : types) { |
| PrimitiveTypeName[] nonInt64 = new PrimitiveTypeName[]{ |
| BOOLEAN, INT32, INT96, DOUBLE, FLOAT, BINARY |
| }; |
| for (final PrimitiveTypeName type : nonInt64) { |
| assertThrows("Should reject non-int64 type: " + type, |
| IllegalStateException.class, (Callable<Type>) () -> Types.required(type).as(logicalType).named("col")); |
| } |
| assertThrows("Should reject non-int64 type: FIXED_LEN_BYTE_ARRAY", |
| IllegalStateException.class, (Callable<Type>) () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(1) |
| .as(logicalType).named("col")); |
| } |
| } |
| |
| @Test |
| public void testIntervalAnnotationRejectsNonFixed() { |
| PrimitiveTypeName[] nonFixed = new PrimitiveTypeName[]{ |
| BOOLEAN, INT32, INT64, INT96, DOUBLE, FLOAT, BINARY |
| }; |
| for (final PrimitiveTypeName type : nonFixed) { |
| assertThrows("Should reject non-fixed type: " + type, |
| IllegalStateException.class, () -> Types.required(type) |
| .as(LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance()).named("interval")); |
| } |
| } |
| |
| @Test |
| public void testIntervalAnnotationRejectsNonFixed12() { |
| assertThrows("Should reject fixed with length != 12: " + 11, |
| IllegalStateException.class, () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(11) |
| .as(LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance()).named("interval")); |
| } |
| |
| @Test |
| public void testTypeConstructionWithUnsupportedColumnOrder() { |
| assertThrows(null, IllegalArgumentException.class, |
| () -> Types.optional(INT96).columnOrder(ColumnOrder.typeDefined()).named("int96_unsupported")); |
| assertThrows(null, IllegalArgumentException.class, |
| () -> Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(12) |
| .as(LogicalTypeAnnotation.IntervalLogicalTypeAnnotation.getInstance()) |
| .columnOrder(ColumnOrder.typeDefined()).named("interval_unsupported")); |
| } |
| |
| @Test |
| public void testDecimalLogicalType() { |
| PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal", |
| LogicalTypeAnnotation.decimalType(3, 4)); |
| PrimitiveType actual = Types.required(BINARY) |
| .as(LogicalTypeAnnotation.decimalType(3, 4)).named("aDecimal"); |
| Assert.assertEquals(expected, actual); |
| } |
| |
| @Test |
| public void testDecimalLogicalTypeWithDeprecatedScale() { |
| PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal", |
| LogicalTypeAnnotation.decimalType(3, 4)); |
| PrimitiveType actual = Types.required(BINARY) |
| .as(LogicalTypeAnnotation.decimalType(3, 4)).scale(3).named("aDecimal"); |
| Assert.assertEquals(expected, actual); |
| } |
| |
| @Test |
| public void testDecimalLogicalTypeWithDeprecatedPrecision() { |
| PrimitiveType expected = new PrimitiveType(REQUIRED, BINARY, "aDecimal", |
| LogicalTypeAnnotation.decimalType(3, 4)); |
| PrimitiveType actual = Types.required(BINARY) |
| .as(LogicalTypeAnnotation.decimalType(3, 4)).precision(4).named("aDecimal"); |
| Assert.assertEquals(expected, actual); |
| } |
| |
| @Test |
| public void testTimestampLogicalTypeWithUTCParameter() { |
| PrimitiveType utcMillisExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp", |
| timestampType(true, MILLIS)); |
| PrimitiveType nonUtcMillisExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp", |
| timestampType(false, MILLIS)); |
| PrimitiveType utcMicrosExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp", |
| timestampType(true, MICROS)); |
| PrimitiveType nonUtcMicrosExpected = new PrimitiveType(REQUIRED, INT64, "aTimestamp", |
| timestampType(false, MICROS)); |
| |
| PrimitiveType utcMillisActual = Types.required(INT64) |
| .as(timestampType(true, MILLIS)).named("aTimestamp"); |
| PrimitiveType nonUtcMillisActual = Types.required(INT64) |
| .as(timestampType(false, MILLIS)).named("aTimestamp"); |
| PrimitiveType utcMicrosActual = Types.required(INT64) |
| .as(timestampType(true, MICROS)).named("aTimestamp"); |
| PrimitiveType nonUtcMicrosActual = Types.required(INT64) |
| .as(timestampType(false, MICROS)).named("aTimestamp"); |
| |
| Assert.assertEquals(utcMillisExpected, utcMillisActual); |
| Assert.assertEquals(nonUtcMillisExpected, nonUtcMillisActual); |
| Assert.assertEquals(utcMicrosExpected, utcMicrosActual); |
| Assert.assertEquals(nonUtcMicrosExpected, nonUtcMicrosActual); |
| } |
| |
| @Test(expected = IllegalArgumentException.class) |
| public void testDecimalLogicalTypeWithDeprecatedScaleMismatch() { |
| Types.required(BINARY) |
| .as(LogicalTypeAnnotation.decimalType(3, 4)) |
| .scale(4).named("aDecimal"); |
| } |
| |
| @Test(expected = IllegalArgumentException.class) |
| public void testDecimalLogicalTypeWithDeprecatedPrecisionMismatch() { |
| Types.required(BINARY) |
| .as(LogicalTypeAnnotation.decimalType(3, 4)) |
| .precision(5).named("aDecimal"); |
| } |
| |
| @Test |
| public void testUUIDLogicalType() { |
| assertEquals( |
| "required fixed_len_byte_array(16) uuid_field (UUID)", |
| Types.required(FIXED_LEN_BYTE_ARRAY).length(16).as(uuidType()).named("uuid_field").toString()); |
| |
| assertThrows("Should fail with invalid length", IllegalStateException.class, |
| () -> Types.required(FIXED_LEN_BYTE_ARRAY).length(10).as(uuidType()).named("uuid_field").toString()); |
| assertThrows("Should fail with invalid type", IllegalStateException.class, |
| () -> Types.required(BINARY).as(uuidType()).named("uuid_field").toString()); |
| } |
| |
| /** |
| * A convenience method to avoid a large number of @Test(expected=...) tests |
| * @param message A String message to describe this assertion |
| * @param expected An Exception class that the Runnable should throw |
| * @param callable A Callable that is expected to throw the exception |
| */ |
| public static void assertThrows( |
| String message, Class<? extends Exception> expected, Callable callable) { |
| try { |
| callable.call(); |
| Assert.fail("No exception was thrown (" + message + "), expected: " + |
| expected.getName()); |
| } catch (Exception actual) { |
| Assert.assertEquals(message, expected, actual.getClass()); |
| } |
| } |
| } |