CRUNCH-465: Fix Avro-related build problems.
Update Avro to 1.7.7 and fix resulting errors.
Fix a brittle integration test.
Signed-off-by: Josh Wills <jwills@apache.org>
diff --git a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java
index ffcc931..04711e4 100644
--- a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java
@@ -17,8 +17,9 @@
*/
package org.apache.crunch;
-import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
import java.util.HashSet;
import java.util.List;
@@ -38,6 +39,7 @@
import org.junit.Rule;
import org.junit.Test;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -87,9 +89,7 @@
Map<String, Long> keywordsMap = countersToMap(result.getStageResults(), KEYWORDS_COUNTER_GROUP);
- assertEquals(3, keywordsMap.size());
-
- assertEquals("{NOT=157, AND=596, OR=81}", keywordsMap.toString());
+ assertThat(keywordsMap, is((Map<String, Long>) ImmutableMap.of("NOT", 157L, "AND", 596L, "OR", 81L)));
}
private static PipelineResult coutSpecialKeywords(Pipeline pipeline, String inputFileName, PTypeFamily tf) {
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
index 6dbb6de..79736b8 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
@@ -22,7 +22,6 @@
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumWriter;
import org.apache.avro.mapred.AvroJob;
import org.apache.avro.mapred.AvroWrapper;
import org.apache.hadoop.conf.Configuration;
@@ -53,7 +52,7 @@
if (org.apache.hadoop.mapred.FileOutputFormat.getCompressOutput(jc)) {
int level = conf.getInt(org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY,
- org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
+ org.apache.avro.file.CodecFactory.DEFAULT_DEFLATE_LEVEL);
String codecName = conf.get(AvroJob.OUTPUT_CODEC,
org.apache.avro.file.DataFileConstants.DEFLATE_CODEC);
CodecFactory codec = codecName.equals(org.apache.avro.file.DataFileConstants.DEFLATE_CODEC)
diff --git a/pom.xml b/pom.xml
index 8250712..2da28a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,7 +73,7 @@
<commons-httpclient.version>3.0.1</commons-httpclient.version>
<commons-logging.version>1.1.1</commons-logging.version>
<commons-cli.version>1.2</commons-cli.version>
- <avro.version>1.7.4</avro.version>
+ <avro.version>1.7.7</avro.version>
<parquet.version>1.3.2</parquet.version>
<javassist.version>3.16.1-GA</javassist.version>
<jackson.version>1.8.8</jackson.version>
