TAJO-2189: Dictionary encoded text in ORC scanner may cause incorrect result. (#1055)
diff --git a/CHANGES b/CHANGES
index 82b8489..37b7af2 100644
--- a/CHANGES
+++ b/CHANGES
@@ -169,6 +169,8 @@
BUG FIXES
+ TAJO-2189: Dictionary encoded text in ORC scanner may cause incorrect result. (jinho)
+
TAJO-2188: Can't start tajo daemon on HDP,CDH (jinho)
TAJO-2184: Can not run Tajo with non-default $TAJO_CONF_DIR (Lee Dongjin via hyunsik)
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java
index 3099779..2f06c48 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/thirdparty/orc/TreeReaderFactory.java
@@ -1217,7 +1217,7 @@
} else {
result.clear();
}
- return result.getBytes();
+ return result.copyBytes();
} else {
return null;
}
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
index deb758d..871bb2d 100644
--- a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -78,6 +78,16 @@
" ]\n" +
"}\n";
+ private static String TEST_VARIABLE_LENGTH_AVRO_SCHEMA =
+ "{\n" +
+ " \"type\": \"record\",\n" +
+ " \"namespace\": \"org.apache.tajo\",\n" +
+ " \"name\": \"testVariableLength\",\n" +
+ " \"fields\": [\n" +
+ " { \"name\": \"col1\", \"type\": \"string\" }\n" +
+ " ]\n" +
+ "}\n";
+
private static String TEST_NULL_HANDLING_TYPES_AVRO_SCHEMA =
"{\n" +
" \"type\": \"record\",\n" +
@@ -480,6 +490,51 @@
}
@Test
+ public void testVariableLength() throws IOException {
+ SchemaBuilder schemaBld = SchemaBuilder.builder()
+ .add("col1", Type.TEXT);
+
+ Schema schema = schemaBld.build();
+
+ TableMeta meta = CatalogUtil.newTableMeta(dataFormat, conf);
+ if (dataFormat.equalsIgnoreCase(BuiltinStorages.AVRO)) {
+ meta.putProperty(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_VARIABLE_LENGTH_AVRO_SCHEMA);
+ }
+
+ FileTablespace sm = TablespaceManager.getLocalFs();
+ Path tablePath = new Path(testDir, "testVariableLength.data");
+ Appender appender = sm.getAppender(meta, schema, tablePath);
+ appender.init();
+
+ String testStr = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
+ String testStr2 = "xxxxxxxxxxxxxxxx"; // test for dictionary encoding
+
+ for(int i = 100; i > 0; i--) {
+ VTuple tuple = new VTuple(1 );
+ tuple.put(new Datum[] {
+ DatumFactory.createText(i % 2 == 0 ? testStr + i : testStr2)
+ });
+
+ appender.addTuple(tuple);
+ }
+ appender.flush();
+ appender.close();
+
+ FileStatus status = fs.getFileStatus(tablePath);
+ FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = sm.getScanner(meta, schema, fragment, null);
+ scanner.init();
+
+ Tuple retrieved;
+ int idx = 100;
+ while ((retrieved = scanner.next()) != null) {
+ assertEquals((idx % 2 == 0 ? testStr + idx : testStr2), retrieved.asDatum(0).toString());
+ idx--;
+ }
+ scanner.close();
+ }
+
+ @Test
public void testNullHandlingTypes() throws IOException {
SchemaBuilder schemaBld = SchemaBuilder.builder()
.add("col1", Type.BOOLEAN)
