PARQUET-1984: Allow tests to run on windows (#870)
Check for \r\n lineendings instead of \n
Change file layout (backslash and slash) to check
Close files / streams before deleting file
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index 4ffd36b..3997808 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -581,10 +581,10 @@
public void testMetadataToJson() {
ParquetMetadata metadata = new ParquetMetadata(null, null);
assertEquals("{\"fileMetaData\":null,\"blocks\":null}", ParquetMetadata.toJSON(metadata));
- assertEquals("{\n" +
+ assertEquals(("{\n" +
" \"fileMetaData\" : null,\n" +
" \"blocks\" : null\n" +
- "}", ParquetMetadata.toPrettyJSON(metadata));
+ "}").replace("\n", System.lineSeparator()), ParquetMetadata.toPrettyJSON(metadata));
}
private ColumnChunkMetaData createColumnChunkMetaData() {
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/DeprecatedInputFormatTest.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/DeprecatedInputFormatTest.java
index 6909354..0662c5c 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/DeprecatedInputFormatTest.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/DeprecatedInputFormatTest.java
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -261,15 +261,17 @@
conf.set(ParquetInputFormat.READ_SUPPORT_CLASS, GroupReadSupport.class.getCanonicalName());
JobClient.runJob(conf);
File partFile = outputDir.listFiles(new PartFileFilter())[0];
- BufferedReader br = new BufferedReader(new FileReader(partFile));
- String line;
- Set<String> s = new HashSet<String>();
- while ((line = br.readLine()) != null) {
- s.add(line.split("\t")[1]);
+ try (BufferedReader br = new BufferedReader(new FileReader(partFile))) {
+ String line;
+ Set<String> s = new HashSet<String>();
+ while ((line = br.readLine()) != null) {
+ s.add(line.split("\t")[1]);
+ }
+ assertEquals(s.size(), 2);
+ assertTrue(s.contains("hello"));
+ assertTrue(s.contains("world"));
}
- assertEquals(s.size(), 2);
- assertTrue(s.contains("hello"));
- assertTrue(s.contains("world"));
+
FileUtils.deleteDirectory(inputDir);
FileUtils.deleteDirectory(outputDir);
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java
index d950316..02c80fc 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputFormat.java
@@ -42,6 +42,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang.SystemUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
@@ -391,23 +392,20 @@
tempDir.deleteOnExit();
int numFiles = 10; // create a nontrivial number of files so that it actually tests getFooters() returns files in the correct order
- StringBuilder url = new StringBuilder();
+ Path[] paths = new Path[numFiles];
for (int i = 0; i < numFiles; i++) {
File file = new File(tempDir, String.format("part-%05d.parquet", i));
createParquetFile(file);
- if (i > 0) {
- url.append(',');
- }
- url.append("file:").append(file.getAbsolutePath());
+ paths[i] = new Path(file.toURI());
}
Job job = new Job();
- FileInputFormat.setInputPaths(job, url.toString());
+ FileInputFormat.setInputPaths(job, paths);
List<Footer> footers = new ParquetInputFormat<Object>().getFooters(job);
for (int i = 0; i < numFiles; i++) {
Footer footer = footers.get(i);
File file = new File(tempDir, String.format("part-%05d.parquet", i));
- assertEquals("file:" + file.getAbsolutePath(), footer.getFile().toString());
+ assertEquals(file.toURI().toString(), footer.getFile().toString());
}
}
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java
index e5cd40f..91e1735 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/statistics/TestStatistics.java
@@ -113,12 +113,11 @@
ParquetProperties.WriterVersion writerVersion = context.version;
CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
- ParquetWriter<Group> writer = new ParquetWriter<Group>(context.fsPath,
+ try (ParquetWriter<Group> writer = new ParquetWriter<Group>(context.fsPath,
groupWriteSupport, codec, blockSize, pageSize, dictionaryPageSize,
- enableDictionary, enableValidation, writerVersion, configuration);
-
- context.write(writer);
- writer.close();
+ enableDictionary, enableValidation, writerVersion, configuration)) {
+ context.write(writer);
+ }
context.test();
@@ -467,17 +466,18 @@
Configuration configuration = new Configuration();
ParquetMetadata metadata = ParquetFileReader.readFooter(configuration,
super.fsPath, ParquetMetadataConverter.NO_FILTER);
- ParquetFileReader reader = new ParquetFileReader(configuration,
+ try (ParquetFileReader reader = new ParquetFileReader(configuration,
metadata.getFileMetaData(),
super.fsPath,
metadata.getBlocks(),
- metadata.getFileMetaData().getSchema().getColumns());
+ metadata.getFileMetaData().getSchema().getColumns())) {
- PageStatsValidator validator = new PageStatsValidator();
+ PageStatsValidator validator = new PageStatsValidator();
- PageReadStore pageReadStore;
- while ((pageReadStore = reader.readNextRowGroup()) != null) {
- validator.validate(metadata.getFileMetaData().getSchema(), pageReadStore);
+ PageReadStore pageReadStore;
+ while ((pageReadStore = reader.readNextRowGroup()) != null) {
+ validator.validate(metadata.getFileMetaData().getSchema(), pageReadStore);
+ }
}
}
}
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/hadoop/thrift/TestParquetToThriftReadWriteAndProjection.java b/parquet-thrift/src/test/java/org/apache/parquet/hadoop/thrift/TestParquetToThriftReadWriteAndProjection.java
index eaef499..3fc71f4 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/hadoop/thrift/TestParquetToThriftReadWriteAndProjection.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/hadoop/thrift/TestParquetToThriftReadWriteAndProjection.java
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -371,15 +371,15 @@
T readValue = null;
for (InputSplit split : splits) {
TaskAttemptContext taskAttemptContext = ContextUtil.newTaskAttemptContext(ContextUtil.getConfiguration(job), new TaskAttemptID(new TaskID(jobID, true, 1), 0));
- final RecordReader<Void, T> reader = parquetThriftInputFormat.createRecordReader(split, taskAttemptContext);
- reader.initialize(split, taskAttemptContext);
- if (reader.nextKeyValue()) {
- readValue = reader.getCurrentValue();
- LOG.info("{}", readValue);
+ try (final RecordReader<Void, T> reader = parquetThriftInputFormat.createRecordReader(split, taskAttemptContext)) {
+ reader.initialize(split, taskAttemptContext);
+ if (reader.nextKeyValue()) {
+ readValue = reader.getCurrentValue();
+ LOG.info("{}", readValue);
+ }
}
}
assertEquals(exptectedReadResult, readValue);
-
}
}
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftMetaData.java b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftMetaData.java
index 64502a5..3c68806 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftMetaData.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftMetaData.java
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -42,12 +42,12 @@
StructType descriptor = new StructType(new ArrayList<ThriftField>(), StructOrUnionType.STRUCT);
ThriftMetaData tmd = new ThriftMetaData("non existent class!!!", descriptor);
- assertEquals("ThriftMetaData(thriftClassName: non existent class!!!, descriptor: {\n" +
+ assertEquals(("ThriftMetaData(thriftClassName: non existent class!!!, descriptor: {\n" +
" \"id\" : \"STRUCT\",\n" +
" \"children\" : [ ],\n" +
" \"structOrUnionType\" : \"STRUCT\",\n" +
" \"logicalTypeAnnotation\" : null\n" +
- "})", tmd.toString());
+ "})").replace("\n", System.lineSeparator()), tmd.toString());
tmd = new ThriftMetaData("non existent class!!!", null);
assertEquals("ThriftMetaData(thriftClassName: non existent class!!!, descriptor: null)", tmd.toString());
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftRecordConverter.java b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftRecordConverter.java
index 7c6d964..d9987ce 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftRecordConverter.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftRecordConverter.java
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -60,7 +60,7 @@
conv.addBinary(Binary.fromString("FAKE_ENUM_VALUE"));
fail("this should throw");
} catch (ParquetDecodingException e) {
- assertEquals("Unrecognized enum value: FAKE_ENUM_VALUE known values: {Binary{\"hello\"}=77} in {\n" +
+ assertEquals(("Unrecognized enum value: FAKE_ENUM_VALUE known values: {Binary{\"hello\"}=77} in {\n" +
" \"name\" : \"name\",\n" +
" \"fieldId\" : 1,\n" +
" \"requirement\" : \"REQUIRED\",\n" +
@@ -72,7 +72,7 @@
" } ],\n" +
" \"logicalTypeAnnotation\" : null\n" +
" }\n" +
- "}", e.getMessage());
+ "}").replace("\n", System.lineSeparator()), e.getMessage());
}
}
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java b/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java
index c24cbc6..0a4566a 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/struct/TestThriftType.java
@@ -32,28 +32,28 @@
@Test
public void testWriteUnionInfo() throws Exception {
StructType st = new StructType(new LinkedList<ThriftField>(), null);
- assertEquals("{\n"
+ assertEquals(("{\n"
+" \"id\" : \"STRUCT\",\n"
+" \"children\" : [ ],\n"
+" \"structOrUnionType\" : \"STRUCT\",\n"
+" \"logicalTypeAnnotation\" : null\n"
- +"}", st.toJSON());
+ +"}").replace("\n", System.lineSeparator()), st.toJSON());
st = new StructType(new LinkedList<ThriftField>(), StructOrUnionType.UNION);
- assertEquals("{\n"
+ assertEquals(("{\n"
+" \"id\" : \"STRUCT\",\n"
+" \"children\" : [ ],\n"
+" \"structOrUnionType\" : \"UNION\",\n"
+" \"logicalTypeAnnotation\" : null\n"
- +"}", st.toJSON());
+ +"}").replace("\n", System.lineSeparator()), st.toJSON());
st = new StructType(new LinkedList<ThriftField>(), StructOrUnionType.STRUCT);
- assertEquals("{\n"
+ assertEquals(("{\n"
+" \"id\" : \"STRUCT\",\n"
+" \"children\" : [ ],\n"
+" \"structOrUnionType\" : \"STRUCT\",\n"
+" \"logicalTypeAnnotation\" : null\n"
- +"}", st.toJSON());
+ +"}").replace("\n", System.lineSeparator()), st.toJSON());
}
@Test
