mnemonic-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject incubator-mnemonic git commit: MNEMONIC-188: Verify all partitions that generated by hadoop writer in test case
Date Tue, 07 Feb 2017 18:19:20 GMT
Repository: incubator-mnemonic
Updated Branches:
  refs/heads/master 12fb0f6c7 -> 4d1afeed1


MNEMONIC-188: Verify all partitions that generated by hadoop writer in test case


Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/4d1afeed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/4d1afeed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/4d1afeed

Branch: refs/heads/master
Commit: 4d1afeed1a4b911f7101a2177330130e4cb013b6
Parents: 12fb0f6
Author: Wang, Gang(Gary) <gang1.wang@intel.com>
Authored: Tue Feb 7 10:08:14 2017 -0800
Committer: Wang, Gang(Gary) <gang1.wang@intel.com>
Committed: Tue Feb 7 10:14:17 2017 -0800

----------------------------------------------------------------------
 .../mnemonic/mapreduce/MneMapreduceIOTest.java  | 36 +++++++++++++-------
 .../org/apache/mnemonic/mapreduce/Person.java   |  4 +--
 2 files changed, 25 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/4d1afeed/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
index 47b3c75..994a42f 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
@@ -53,7 +53,7 @@ public class MneMapreduceIOTest {
   private Random m_rand;
   private TaskAttemptID m_taid;
   private TaskAttemptContext m_tacontext;
-  private long m_reccnt = 5000L;
+  private long m_reccnt = 500000L;
   private long m_sumage = 0L;
 
   @BeforeClass
@@ -112,20 +112,30 @@ public class MneMapreduceIOTest {
   @Test(enabled = true, dependsOnMethods = { "testWritePersonData" })
   public void testReadPersonData() throws Exception {
     long sumage = 0L;
-    FileSplit split = new FileSplit(
-        new Path(m_workdir,
-            String.format("part-00001-m-00000%s", MneConfigHelper.FILE_EXTENSION)), 0, 0L,
new String[0]);
-    InputFormat<NullWritable, Person<Long>> inputFormat = new MneInputFormat<Person<Long>>();
-    RecordReader<NullWritable, Person<Long>> reader = inputFormat.createRecordReader(split,
m_tacontext);
-    Person<Long> person = null;
-    for (int i = 0; i < m_reccnt; ++i) {
-      AssertJUnit.assertTrue(reader.nextKeyValue());
-      person = reader.getCurrentValue();
-      AssertJUnit.assertTrue(person.getAge() < 51);
-      sumage += person.getAge();
+    long reccnt = 0L;
+    File folder = new File(m_workdir.toString());
+    File[] listfiles = folder.listFiles();
+    for (int idx = 0; idx < listfiles.length; ++idx) {
+      if (listfiles[idx].isFile()
+          && listfiles[idx].getName().endsWith(MneConfigHelper.FILE_EXTENSION)) {
+        System.out.println(String.format("Verifying : %s", listfiles[idx].getName()));
+        FileSplit split = new FileSplit(
+            new Path(m_workdir, listfiles[idx].getName()), 0, 0L, new String[0]);
+        InputFormat<NullWritable, Person<Long>> inputFormat = new MneInputFormat<Person<Long>>();
+        RecordReader<NullWritable, Person<Long>> reader = inputFormat.createRecordReader(
+            split, m_tacontext);
+        Person<Long> person = null;
+        while (reader.nextKeyValue()) {
+          person = reader.getCurrentValue();
+          AssertJUnit.assertTrue(person.getAge() < 51);
+          sumage += person.getAge();
+          ++reccnt;
+        }
+        reader.close();
+      }
     }
     AssertJUnit.assertEquals(m_sumage, sumage);
-    reader.close();
+    AssertJUnit.assertEquals(m_reccnt, reccnt);
     System.out.println(String.format("The sum of ages is %d", sumage));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/4d1afeed/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
index 17dbc5c..c5edd49 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/Person.java
@@ -42,12 +42,12 @@ public abstract class Person<E> implements Durable, Comparable<Person<E>>,
Writa
 
   @Override
   public void initializeAfterCreate() {
-    System.out.println("Initializing After Created");
+    /* System.out.println("Initializing After Created"); */
   }
 
   @Override
   public void initializeAfterRestore() {
-    System.out.println("Initializing After Restored");
+    /* System.out.println("Initializing After Restored"); */
   }
 
   @Override


Mime
View raw message