mnemonic-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject incubator-mnemonic git commit: MNEMONIC-192: Optimize the code of MneConfigHelper
Date Tue, 14 Feb 2017 19:39:50 GMT
Repository: incubator-mnemonic
Updated Branches:
  refs/heads/master c4e7a33c1 -> d8338f3dc


MNEMONIC-192: Optimize the code of MneConfigHelper


Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/d8338f3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/d8338f3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/d8338f3d

Branch: refs/heads/master
Commit: d8338f3dc0789974c63683351a2faa87111ac983
Parents: c4e7a33
Author: Wang, Gang(Gary) <gang1.wang@intel.com>
Authored: Tue Feb 14 11:31:01 2017 -0800
Committer: Wang, Gang(Gary) <gang1.wang@intel.com>
Committed: Tue Feb 14 11:31:01 2017 -0800

----------------------------------------------------------------------
 .../apache/mnemonic/hadoop/MneConfigHelper.java | 133 +++++--------------
 .../mapreduce/MneMapreduceRecordReader.java     |   8 +-
 .../mapreduce/MneMapreduceRecordWriter.java     |  11 +-
 .../mnemonic/mapreduce/MneMapreduceIOTest.java  |  27 ++--
 4 files changed, 57 insertions(+), 122 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/d8338f3d/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/MneConfigHelper.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/MneConfigHelper.java
b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/MneConfigHelper.java
index 7bbcddb..be78709 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/MneConfigHelper.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/MneConfigHelper.java
@@ -33,44 +33,30 @@ import org.slf4j.LoggerFactory;
  */
 public class MneConfigHelper {
 
-  private static final String INPUT_DURABLE_TYPES_CONFIG = "mnemonic.input.durable.types";
-  private static final String INPUT_ENTITY_FACTORY_PROXIES = "mnemonic.input.entity.factory.proxies.class";
-  private static final String OUTPUT_DURABLE_TYPES_CONFIG = "mnemonic.output.durable.types";
-  private static final String OUTPUT_ENTITY_FACTORY_PROXIES = "mnemonic.output.entity.factory.proxies.class";
-  private static final String INPUT_SLOT_KEY_ID = "mnemonic.input.slot.key.id";
-  private static final String OUTPUT_SLOT_KEY_ID = "mnemonic.output.slot.key.id";
-  private static final String INPUT_MEM_SERVICE_NAME = "mnemonic.input.mem.service.name";
-  private static final String OUTPUT_MEM_SERVICE_NAME = "mnemonic.output.mem.service.name";
-  private static final String OUTPUT_MEM_POOL_SIZE = "mnemonic.output.mem.pool.size";
-//  private static final String RECORD_FACTORY_CLASS = "mnemonic.record_factory_class";
+  public static final String OUTPUT_CONFIG_PREFIX_DEFAULT = "mnemonic.output.";
+  public static final String INPUT_CONFIG_PREFIX_DEFAULT = "mnemonic.input.";
+  public static final String DURABLE_TYPES = "durable.types";
+  public static final String ENTITY_FACTORY_PROXIES = "entity.factory.proxies.class";
+  public static final String SLOT_KEY_ID = "slot.key.id";
+  public static final String MEM_SERVICE_NAME = "mem.service.name";
+  public static final String MEM_POOL_SIZE = "mem.pool.size";
   private static final long DEFAULT_OUTPUT_MEM_POOL_SIZE = 1024L * 1024 * 1024 * 4;
   public static final String FILE_EXTENSION = ".mne";
 
   private static final Logger LOGGER = LoggerFactory.getLogger(MneConfigHelper.class);
 
-  public static void setInputDurableTypes(Configuration conf, DurableType[] dtypes) {
-    String val = StringUtils.join(dtypes, ",");
-    conf.set(INPUT_DURABLE_TYPES_CONFIG, val);
-  }
-
-  public static DurableType[] getInputDurableTypes(Configuration conf) {
-    List<DurableType> ret = new ArrayList<>();
-    String val = conf.get(INPUT_DURABLE_TYPES_CONFIG);
-    String[] vals = StringUtils.split(val, ",");
-    for (String itm : vals) {
-      ret.add(DurableType.valueOf(itm));
-    }
-    return ret.toArray(new DurableType[0]);
+  public static String getConfigName(String prefix, String partname) {
+    return prefix + partname;
   }
 
-  public static void setOutputDurableTypes(Configuration conf, DurableType[] dtypes) {
+  public static void setDurableTypes(Configuration conf, String prefix, DurableType[] dtypes)
{
     String val = StringUtils.join(dtypes, ",");
-    conf.set(OUTPUT_DURABLE_TYPES_CONFIG, val);
+    conf.set(getConfigName(prefix, DURABLE_TYPES), val);
   }
 
-  public static DurableType[] getOutputDurableTypes(Configuration conf) {
+  public static DurableType[] getDurableTypes(Configuration conf, String prefix) {
     List<DurableType> ret = new ArrayList<>();
-    String val = conf.get(OUTPUT_DURABLE_TYPES_CONFIG);
+    String val = conf.get(getConfigName(prefix, DURABLE_TYPES));
     String[] vals = StringUtils.split(val, ",");
     for (String itm : vals) {
       ret.add(DurableType.valueOf(itm));
@@ -78,40 +64,17 @@ public class MneConfigHelper {
     return ret.toArray(new DurableType[0]);
   }
 
-  public static void setInputEntityFactoryProxies(Configuration conf, Class<?>[] proxies)
{
-    List<String> vals = new ArrayList<>();
-    for (Class<?> itm : proxies) {
-      vals.add(itm.getName());
-    }
-    conf.setStrings(INPUT_ENTITY_FACTORY_PROXIES, vals.toArray(new String[0]));
-  }
-
-  public static Class<?>[] getInputEntityFactoryProxies(Configuration conf) {
-    List<Class<?>> ret = new ArrayList<>();
-    String[] vals = conf.getStrings(INPUT_ENTITY_FACTORY_PROXIES);
-    String clsname = null;
-    try {
-      for (String itm : vals) {
-        clsname = itm;
-        ret.add(Class.forName(itm));
-      }
-    } catch (ClassNotFoundException | NoClassDefFoundError e) {
-      throw new RuntimeException(String.format("Unable to find class '%s'", clsname), e);
-    }
-    return ret.toArray(new Class<?>[0]);
-  }
-
-  public static void setOutputEntityFactoryProxies(Configuration conf, Class<?>[] proxies)
{
+  public static void setEntityFactoryProxies(Configuration conf, String prefix, Class<?>[]
proxies) {
     List<String> vals = new ArrayList<>();
     for (Class<?> itm : proxies) {
       vals.add(itm.getName());
     }
-    conf.setStrings(OUTPUT_ENTITY_FACTORY_PROXIES, vals.toArray(new String[0]));
+    conf.setStrings(getConfigName(prefix, ENTITY_FACTORY_PROXIES), vals.toArray(new String[0]));
   }
 
-  public static Class<?>[] getOutputEntityFactoryProxies(Configuration conf) {
+  public static Class<?>[] getEntityFactoryProxies(Configuration conf, String prefix)
{
     List<Class<?>> ret = new ArrayList<>();
-    String[] vals = conf.getStrings(OUTPUT_ENTITY_FACTORY_PROXIES);
+    String[] vals = conf.getStrings(getConfigName(prefix, ENTITY_FACTORY_PROXIES));
     String clsname = null;
     try {
       for (String itm : vals) {
@@ -124,68 +87,32 @@ public class MneConfigHelper {
     return ret.toArray(new Class<?>[0]);
   }
 
-  public static void setInputSlotKeyId(Configuration conf, long keyid) {
-    conf.setLong(INPUT_SLOT_KEY_ID, keyid);
-  }
-
-  public static long getInputSlotKeyId(Configuration conf) {
-    return conf.getLong(INPUT_SLOT_KEY_ID, 0L);
+  public static void setSlotKeyId(Configuration conf, String prefix, long keyid) {
+    conf.setLong(getConfigName(prefix, SLOT_KEY_ID), keyid);
   }
 
-  public static void setOutputSlotKeyId(Configuration conf, long keyid) {
-    conf.setLong(OUTPUT_SLOT_KEY_ID, keyid);
+  public static long getSlotKeyId(Configuration conf, String prefix) {
+    return conf.getLong(getConfigName(prefix, SLOT_KEY_ID), 0L);
   }
 
-  public static long getOutputSlotKeyId(Configuration conf) {
-    return conf.getLong(OUTPUT_SLOT_KEY_ID, 0L);
+  public static void setMemServiceName(Configuration conf, String prefix, String name) {
+    conf.set(getConfigName(prefix, MEM_SERVICE_NAME), name);
   }
 
-  public static void setInputMemServiceName(Configuration conf, String name) {
-    conf.set(INPUT_MEM_SERVICE_NAME, name);
-  }
-
-  public static String getInputMemServiceName(Configuration conf) {
-    String ret = conf.get(INPUT_MEM_SERVICE_NAME);
-    if (null == ret) {
-      throw new ConfigurationException("You must set the input mem service name");
-    }
-    return ret;
-  }
-
-  public static void setOutputMemServiceName(Configuration conf, String name) {
-    conf.set(OUTPUT_MEM_SERVICE_NAME, name);
-  }
-
-  public static String getOutputMemServiceName(Configuration conf) {
-    String ret = conf.get(OUTPUT_MEM_SERVICE_NAME);
+  public static String getMemServiceName(Configuration conf, String prefix) {
+    String ret = conf.get(getConfigName(prefix, MEM_SERVICE_NAME));
     if (null == ret) {
-      throw new ConfigurationException("You must set the output mem service name");
+      throw new ConfigurationException("You must set the mem service name");
     }
     return ret;
   }
 
-  public static void setOutputMemPoolSize(Configuration conf, long size) {
-    conf.setLong(OUTPUT_MEM_POOL_SIZE, size);
+  public static void setMemPoolSize(Configuration conf, String prefix, long size) {
+    conf.setLong(getConfigName(prefix, MEM_POOL_SIZE), size);
   }
 
-  public static long getOutputMemPoolSize(Configuration conf) {
-    return conf.getLong(OUTPUT_MEM_POOL_SIZE, DEFAULT_OUTPUT_MEM_POOL_SIZE);
+  public static long getMemPoolSize(Configuration conf, String prefix) {
+    return conf.getLong(getConfigName(prefix, MEM_POOL_SIZE), DEFAULT_OUTPUT_MEM_POOL_SIZE);
   }
 
-//
-//  public static Class<?> getRecordFactory(Configuration conf) {
-//    Class<?> ret;
-//    String clsname = conf.get(RECORD_FACTORY_CLASS);
-//    try {
-//      ret = Class.forName(clsname);
-//    } catch (ClassNotFoundException | NoClassDefFoundError e) {
-//      throw new RuntimeException(String.format("Unable to find record factory class '%s'",
clsname), e);
-//    }
-//    return ret;
-//  }
-//
-//  public static void setRecordFactory(Configuration conf, Class<?> recf) {
-//    conf.setStrings(RECORD_FACTORY_CLASS, recf.getName());
-//  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/d8338f3d/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordReader.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordReader.java
b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordReader.java
index bdf6170..3743916 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordReader.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordReader.java
@@ -65,11 +65,11 @@ public class MneMapreduceRecordReader<V>
     FileSplit split = (FileSplit) inputSplit;
     m_context = context;
     m_conf = m_context.getConfiguration();
-    m_msvrname = MneConfigHelper.getInputMemServiceName(m_conf);
-    m_gtypes = MneConfigHelper.getInputDurableTypes(m_conf);
+    m_msvrname = MneConfigHelper.getMemServiceName(m_conf, MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT);
+    m_gtypes = MneConfigHelper.getDurableTypes(m_conf, MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT);
     m_efproxies = Utils.instantiateEntityFactoryProxies(
-        MneConfigHelper.getInputEntityFactoryProxies(m_conf));
-    m_slotkeyid = MneConfigHelper.getInputSlotKeyId(m_conf);
+        MneConfigHelper.getEntityFactoryProxies(m_conf, MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT));
+    m_slotkeyid = MneConfigHelper.getSlotKeyId(m_conf, MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT);
     
     DurableSinglyLinkedList<V> dsllist;
 

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/d8338f3d/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordWriter.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordWriter.java
b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordWriter.java
index c74590f..242ee46 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordWriter.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/main/java/org/apache/mnemonic/hadoop/mapreduce/MneMapreduceRecordWriter.java
@@ -71,12 +71,13 @@ public class MneMapreduceRecordWriter<V> extends RecordWriter<NullWritable,
V>
 
   protected MneMapreduceRecordWriter(Configuration conf) {
     m_conf = conf;
-    m_msvrname = MneConfigHelper.getOutputMemServiceName(m_conf);
-    m_gtypes = MneConfigHelper.getOutputDurableTypes(m_conf);
-    m_efproxies = Utils.instantiateEntityFactoryProxies(MneConfigHelper.getOutputEntityFactoryProxies(m_conf));
+    m_msvrname = MneConfigHelper.getMemServiceName(m_conf, MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT);
+    m_gtypes = MneConfigHelper.getDurableTypes(m_conf, MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT);
+    m_efproxies = Utils.instantiateEntityFactoryProxies(
+        MneConfigHelper.getEntityFactoryProxies(m_conf, MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT));
     m_recparmpair = Utils.shiftDurableParams(m_gtypes, m_efproxies, 1);
-    m_slotkeyid = MneConfigHelper.getOutputSlotKeyId(m_conf);
-    m_poolsz = MneConfigHelper.getOutputMemPoolSize(conf);
+    m_slotkeyid = MneConfigHelper.getSlotKeyId(m_conf, MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT);
+    m_poolsz = MneConfigHelper.getMemPoolSize(m_conf, MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT);
     m_recordmap = new HashMap<V, DurableSinglyLinkedList<V>>();
     if (m_gtypes.length < 1) {
       throw new ConfigurationException("The durable type of record parameters does not exist");

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/d8338f3d/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
----------------------------------------------------------------------
diff --git a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
index 994a42f..33e7697 100644
--- a/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
+++ b/mnemonic-hadoop/mnemonic-hadoop-mapreduce/src/test/java/org/apache/mnemonic/mapreduce/MneMapreduceIOTest.java
@@ -47,6 +47,9 @@ import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 public class MneMapreduceIOTest {
+
+  private static final String SERVICE_NAME = "pmalloc";
+  private static final long SLOT_KEY_ID = 5L;
   private Path m_workdir;
   private JobConf m_conf;
   private FileSystem m_fs;
@@ -76,16 +79,20 @@ public class MneMapreduceIOTest {
 
     m_conf.set("mapreduce.output.fileoutputformat.outputdir", m_workdir.toString());
 
-    MneConfigHelper.setInputMemServiceName(m_conf, "pmalloc");
-    MneConfigHelper.setInputSlotKeyId(m_conf, 3L);
-    MneConfigHelper.setInputDurableTypes(m_conf, new DurableType[] {DurableType.DURABLE});
-    MneConfigHelper.setInputEntityFactoryProxies(m_conf, new Class<?>[] {PersonListEFProxy.class});
-    MneConfigHelper.setOutputMemServiceName(m_conf, "pmalloc");
-    MneConfigHelper.setOutputSlotKeyId(m_conf, 3L);
-    MneConfigHelper.setOutputMemPoolSize(m_conf, 1024L * 1024 * 1024 * 4);
-    MneConfigHelper.setOutputDurableTypes(m_conf, new DurableType[] {DurableType.DURABLE});
-    MneConfigHelper.setOutputEntityFactoryProxies(m_conf, new Class<?>[] {PersonListEFProxy.class});
-
+    MneConfigHelper.setMemServiceName(m_conf, MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT,
SERVICE_NAME);
+    MneConfigHelper.setSlotKeyId(m_conf, MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT, SLOT_KEY_ID);
+    MneConfigHelper.setDurableTypes(m_conf,
+        MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT, new DurableType[] {DurableType.DURABLE});
+    MneConfigHelper.setEntityFactoryProxies(m_conf,
+        MneConfigHelper.INPUT_CONFIG_PREFIX_DEFAULT, new Class<?>[] {PersonListEFProxy.class});
+    MneConfigHelper.setMemServiceName(m_conf, MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT,
SERVICE_NAME);
+    MneConfigHelper.setSlotKeyId(m_conf, MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT, SLOT_KEY_ID);
+    MneConfigHelper.setMemPoolSize(m_conf,
+        MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT, 1024L * 1024 * 1024 * 4);
+    MneConfigHelper.setDurableTypes(m_conf,
+        MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT, new DurableType[] {DurableType.DURABLE});
+    MneConfigHelper.setEntityFactoryProxies(m_conf,
+        MneConfigHelper.OUTPUT_CONFIG_PREFIX_DEFAULT, new Class<?>[] {PersonListEFProxy.class});
   }
 
   @AfterClass


Mime
View raw message