MNEMONIC-188: Verify all partitions that generated by hadoop writer in test case
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
index 47b3c75..994a42f 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
@@ -53,7 +53,7 @@
private Random m_rand;
private TaskAttemptID m_taid;
private TaskAttemptContext m_tacontext;
- private long m_reccnt = 5000L;
+ private long m_reccnt = 500000L;
private long m_sumage = 0L;
@BeforeClass
@@ -112,20 +112,30 @@
@Test(enabled = true, dependsOnMethods = { "testWritePersonData" })
public void testReadPersonData() throws Exception {
long sumage = 0L;
- FileSplit split = new FileSplit(
- new Path(m_workdir,
- String.format("part-00001-m-00000%s", MneConfigHelper.FILE_EXTENSION)), 0, 0L, new String[0]);
- InputFormat<NullWritable, Person<Long>> inputFormat = new MneInputFormat<Person<Long>>();
- RecordReader<NullWritable, Person<Long>> reader = inputFormat.createRecordReader(split, m_tacontext);
- Person<Long> person = null;
- for (int i = 0; i < m_reccnt; ++i) {
- AssertJUnit.assertTrue(reader.nextKeyValue());
- person = reader.getCurrentValue();
- AssertJUnit.assertTrue(person.getAge() < 51);
- sumage += person.getAge();
+ long reccnt = 0L;
+ File folder = new File(m_workdir.toString());
+ File[] listfiles = folder.listFiles();
+ for (int idx = 0; idx < listfiles.length; ++idx) {
+ if (listfiles[idx].isFile()
+ && listfiles[idx].getName().endsWith(MneConfigHelper.FILE_EXTENSION)) {
+ System.out.println(String.format("Verifying : %s", listfiles[idx].getName()));
+ FileSplit split = new FileSplit(
+ new Path(m_workdir, listfiles[idx].getName()), 0, 0L, new String[0]);
+ InputFormat<NullWritable, Person<Long>> inputFormat = new MneInputFormat<Person<Long>>();
+ RecordReader<NullWritable, Person<Long>> reader = inputFormat.createRecordReader(
+ split, m_tacontext);
+ Person<Long> person = null;
+ while (reader.nextKeyValue()) {
+ person = reader.getCurrentValue();
+ AssertJUnit.assertTrue(person.getAge() < 51);
+ sumage += person.getAge();
+ ++reccnt;
+ }
+ reader.close();
+ }
}
AssertJUnit.assertEquals(m_sumage, sumage);
- reader.close();
+ AssertJUnit.assertEquals(m_reccnt, reccnt);
System.out.println(String.format("The sum of ages is %d", sumage));
}
}
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
index 17dbc5c..c5edd49 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
@@ -42,12 +42,12 @@
@Override
public void initializeAfterCreate() {
- System.out.println("Initializing After Created");
+ /* System.out.println("Initializing After Created"); */
}
@Override
public void initializeAfterRestore() {
- System.out.println("Initializing After Restored");
+ /* System.out.println("Initializing After Restored"); */
}
@Override
