mnemonic-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject [09/14] incubator-mnemonic git commit: MNEMONIC-158: Add durable native computing sorting workload for bench workload
Date Mon, 21 Nov 2016 17:32:35 GMT
MNEMONIC-158: Add durable native computing sorting workload for bench workload


Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/53bf3b67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/53bf3b67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/53bf3b67

Branch: refs/heads/master
Commit: 53bf3b67df91a8b12097465fe74109dba90334b4
Parents: 052f713
Author: Wang, Gang(Gary) <gang1.wang@intel.com>
Authored: Fri Nov 11 16:04:36 2016 -0800
Committer: Wang, Gang(Gary) <gang1.wang@intel.com>
Committed: Thu Nov 17 12:51:14 2016 -0800

----------------------------------------------------------------------
 mnemonic-benches/mnemonic-sort-bench/pom.xml    | 14 ++++
 .../apache/mnemonic/bench/DNCSTextFileSort.java | 80 +++++++++++++++++++-
 .../mnemonic/bench/RegularTestFileSort.java     | 30 +++++++-
 .../java/org/apache/mnemonic/bench/Sort.java    | 16 +++-
 .../org/apache/mnemonic/bench/TextFileSort.java |  8 +-
 mnemonic-benches/pom.xml                        |  5 ++
 .../src/main/native/common.c                    |  8 +-
 7 files changed, 143 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/53bf3b67/mnemonic-benches/mnemonic-sort-bench/pom.xml
----------------------------------------------------------------------
diff --git a/mnemonic-benches/mnemonic-sort-bench/pom.xml b/mnemonic-benches/mnemonic-sort-bench/pom.xml
index b4aed55..32aed1e 100644
--- a/mnemonic-benches/mnemonic-sort-bench/pom.xml
+++ b/mnemonic-benches/mnemonic-sort-bench/pom.xml
@@ -93,6 +93,20 @@
                 </artifactItem>
                 <artifactItem>
                   <groupId>${project.groupId}</groupId>
+                  <artifactId>mnemonic-sys-vmem-service</artifactId>
+                  <version>${project.version}</version>
+                  <classifier>${os.detected.classifier}</classifier>
+                  <type>jar</type>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
+                  <artifactId>mnemonic-nvml-pmem-service</artifactId>
+                  <version>${project.version}</version>
+                  <classifier>${os.detected.classifier}</classifier>
+                  <type>jar</type>
+                </artifactItem>
+                <artifactItem>
+                  <groupId>${project.groupId}</groupId>
                   <artifactId>mnemonic-pmalloc-service</artifactId>
                   <version>${project.version}</version>
                   <classifier>${os.detected.classifier}</classifier>

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/53bf3b67/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/DNCSTextFileSort.java
----------------------------------------------------------------------
diff --git a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/DNCSTextFileSort.java
b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/DNCSTextFileSort.java
index 9f15f84..e2fe132 100644
--- a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/DNCSTextFileSort.java
+++ b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/DNCSTextFileSort.java
@@ -20,22 +20,94 @@ package org.apache.mnemonic.bench;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.mnemonic.VolatileMemAllocator;
+import org.apache.mnemonic.EntityFactoryProxy;
+import org.apache.mnemonic.Utils;
+import org.apache.mnemonic.DurableType;
+import org.apache.mnemonic.collections.DurableSinglyLinkedList;
+import org.apache.mnemonic.collections.DurableSinglyLinkedListFactory;
+import org.apache.mnemonic.service.computingservice.GeneralComputingService;
+import org.apache.mnemonic.service.computingservice.ValueInfo;
 
 public class DNCSTextFileSort implements TextFileSort {
+  private String uri = "";
+  private VolatileMemAllocator m_act = null;
+  private DurableType[] elem_gftypes = {DurableType.LONG};
+  private EntityFactoryProxy[] elem_efproxies = null;
+  private Long head;
+  private long[] sortinfo = new long[3];
+  private long[][] ostack;
 
-  @Override
-  public void build(BufferedReader reader) throws NumberFormatException, IOException {
+  public DNCSTextFileSort() {
+  }
 
+  @Override
+  public void load(BufferedReader reader) throws NumberFormatException, IOException {
+    if (null != m_act) {
+      clear();
+    }
+    m_act = new VolatileMemAllocator(Utils.getVolatileMemoryAllocatorService("sysvmem"),
1024 * 1024 * 1024 * 5,
+        uri, true);
+    String text = null;
+    DurableSinglyLinkedList<Long> curnode = null, prvnode = null, node = null;
+    Long val;
+    while ((text = reader.readLine()) != null) {
+      val = Long.parseLong(text);
+      curnode = DurableSinglyLinkedListFactory.create(m_act, elem_efproxies, elem_gftypes,
false);
+      curnode.setItem(val, false);
+      if (null == prvnode) {
+        this.head = curnode.getHandler();
+        this.ostack = curnode.getNativeFieldInfo();
+      } else {
+        prvnode.setNext(curnode, false);
+      }
+      prvnode = curnode;
+    }
   }
 
   @Override
   public void doSort() {
-
+    GeneralComputingService gcsvr = Utils.getGeneralComputingService("sort");
+    ValueInfo vinfo = new ValueInfo();
+    List<long[][]> objstack = new ArrayList<long[][]>();
+    objstack.add(this.ostack);
+    long[][] fidinfostack = {{2L, 1L}};
+    vinfo.handler = this.head;
+    vinfo.transtable = m_act.getTranslateTable();
+    vinfo.dtype = DurableType.LONG;
+    vinfo.frames = Utils.genNativeParamForm(objstack, fidinfostack);
+    ValueInfo[] vinfos = {vinfo};
+    long[] ret = gcsvr.perform("1dlong_bubble", vinfos);
+    this.head = ret[0];
+    this.sortinfo[0] = ret[1];
+    this.sortinfo[1] = ret[2];
+    this.sortinfo[2] = ret[3];
   }
 
   @Override
-  public void save(BufferedWriter writer) throws IOException {
+  public void store(BufferedWriter writer) throws IOException {
+    DurableSinglyLinkedList<Long> linkvals = DurableSinglyLinkedListFactory.restore(m_act,
+        elem_efproxies, elem_gftypes, this.head, false);
+    Iterator<Long> elemiter = linkvals.iterator();
+    Long val;
+    while (elemiter.hasNext()) {
+      val = elemiter.next();
+      writer.write(val.toString());
+      writer.newLine();
+    }
+  }
 
+  @Override
+  public long[] getSortInfo() {
+    return this.sortinfo;
   }
 
+  @Override
+  public void clear() {
+    m_act.close();
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/53bf3b67/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/RegularTestFileSort.java
----------------------------------------------------------------------
diff --git a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/RegularTestFileSort.java
b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/RegularTestFileSort.java
index 2fe77de..536e362 100644
--- a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/RegularTestFileSort.java
+++ b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/RegularTestFileSort.java
@@ -24,12 +24,13 @@ import java.io.IOException;
 public class RegularTestFileSort implements TextFileSort {
 
   private Node<Long> head;
+  private long[] sortinfo = new long[3];
   
   public RegularTestFileSort() {
   }
 
   @Override
-  public void build(BufferedReader reader) throws NumberFormatException, IOException {
+  public void load(BufferedReader reader) throws NumberFormatException, IOException {
     String text = null;
     Node<Long> curnode = null;
     Long val;
@@ -44,15 +45,21 @@ public class RegularTestFileSort implements TextFileSort {
       }
     }
   }
+
   @Override
   public void doSort() {
     Node<Long> curnode, tmpnode, prevnode;
+    long cntscan = 0L, cntswap = 0L, cntnoswap = 0L;
     boolean changed;
+    if (null == this.head) {
+      return;
+    }
     do {
+      ++cntscan;
       curnode = this.head;
       prevnode = null;
       changed = false;
-      while (null != curnode) {
+      while (true) {
         tmpnode = curnode.getNext();
         if (null == tmpnode) {
           break;
@@ -67,16 +74,22 @@ public class RegularTestFileSort implements TextFileSort {
           }
           prevnode = tmpnode;
           changed = true;
+          ++cntswap;
         } else {
           prevnode = curnode;
           curnode = tmpnode;
+          ++cntnoswap;
         }
       }
     } while (changed);
+    this.sortinfo[0] = cntscan;
+    this.sortinfo[1] = cntswap;
+    this.sortinfo[2] = cntnoswap;
   }
+
   @Override
-  public void save(BufferedWriter writer) throws IOException {
-    Node<Long> curnode = head;
+  public void store(BufferedWriter writer) throws IOException {
+    Node<Long> curnode = this.head;
     while (null != curnode) {
       writer.write(curnode.getData().toString());
       writer.newLine();
@@ -84,4 +97,13 @@ public class RegularTestFileSort implements TextFileSort {
     }
   }
 
+  @Override
+  public long[] getSortInfo() {
+    return this.sortinfo;
+  }
+
+  @Override
+  public void clear() {
+    this.head = null;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/53bf3b67/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/Sort.java
----------------------------------------------------------------------
diff --git a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/Sort.java
b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/Sort.java
index df96058..daca7da 100644
--- a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/Sort.java
+++ b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/Sort.java
@@ -110,14 +110,16 @@ public class Sort {
       }
 
       sttime = System.nanoTime();
-      tfsorter.build(reader);
-      reportElapse("Build Time", sttime, System.nanoTime());
+      tfsorter.load(reader);
+      reportElapse("Load Time", sttime, System.nanoTime());
       sttime = System.nanoTime();
       tfsorter.doSort();
       reportElapse("Sort Time", sttime, System.nanoTime());
       sttime = System.nanoTime();
-      tfsorter.save(writer);
-      reportElapse("Save Time", sttime, System.nanoTime());
+      tfsorter.store(writer);
+      reportElapse("Store Time", sttime, System.nanoTime());
+      reportSortInfo(tfsorter.getSortInfo());
+      tfsorter.clear();
     } catch (FileNotFoundException e) {
       System.err.println(e.getMessage());
       throw e;
@@ -143,4 +145,10 @@ public class Sort {
     System.out.println(String.format("%s : %,d ms.", msg,
         TimeUnit.NANOSECONDS.toMillis(t2 - t1)));
   }
+
+  static void reportSortInfo(long[] sortinfo) {
+    System.out.println(String.format("Scan Count: %,d ", sortinfo[0]));
+    System.out.println(String.format("Swap Count: %,d ", sortinfo[1]));
+    System.out.println(String.format("No Swap Count : %,d ", sortinfo[2]));
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/53bf3b67/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/TextFileSort.java
----------------------------------------------------------------------
diff --git a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/TextFileSort.java
b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/TextFileSort.java
index b88d893..88cd0de 100644
--- a/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/TextFileSort.java
+++ b/mnemonic-benches/mnemonic-sort-bench/src/main/java/org/apache/mnemonic/bench/TextFileSort.java
@@ -23,9 +23,13 @@ import java.io.IOException;
 
 public interface TextFileSort {
 
-  void build(BufferedReader reader) throws NumberFormatException, IOException;
+  void load(BufferedReader reader) throws NumberFormatException, IOException;
 
   void doSort();
 
-  void save(BufferedWriter writer) throws IOException;
+  void store(BufferedWriter writer) throws IOException;
+
+  long[] getSortInfo();
+
+  void clear();
 }

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/53bf3b67/mnemonic-benches/pom.xml
----------------------------------------------------------------------
diff --git a/mnemonic-benches/pom.xml b/mnemonic-benches/pom.xml
index 9c5a664..377f1c0 100644
--- a/mnemonic-benches/pom.xml
+++ b/mnemonic-benches/pom.xml
@@ -49,6 +49,11 @@
       <version>${project.version}</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.mnemonic</groupId>
+      <artifactId>mnemonic-collections</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
       <groupId>commons-cli</groupId>
       <artifactId>commons-cli</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/53bf3b67/mnemonic-computing-services/mnemonic-utilities-service/src/main/native/common.c
----------------------------------------------------------------------
diff --git a/mnemonic-computing-services/mnemonic-utilities-service/src/main/native/common.c
b/mnemonic-computing-services/mnemonic-utilities-service/src/main/native/common.c
index 7f7a35c..1bf50d1 100644
--- a/mnemonic-computing-services/mnemonic-utilities-service/src/main/native/common.c
+++ b/mnemonic-computing-services/mnemonic-utilities-service/src/main/native/common.c
@@ -276,7 +276,7 @@ jlongArray constructJLongArray(JNIEnv* env, long arr[], size_t sz) {
 inline void *to_e(JNIEnv* env, struct NValueInfo *nvinfo, long p) {
   size_t i;
   struct transitem * ti;
-  if (NULL != nvinfo || NULL != nvinfo->transtable) {
+  if (NULL != nvinfo && NULL != nvinfo->transtable) {
     for (i = 0; i < nvinfo->transtablesz; ++i) {
       ti = nvinfo->transtable + i;
       if (p >= ti->hdlbase && p < ti->size) {
@@ -285,7 +285,7 @@ inline void *to_e(JNIEnv* env, struct NValueInfo *nvinfo, long p) {
     }
     throw(env, "No item found in Translate Table.");
   } else {
-    return p;
+    return addr_from_java(p);
   }
   return NULL;
 }
@@ -293,7 +293,7 @@ inline void *to_e(JNIEnv* env, struct NValueInfo *nvinfo, long p) {
 inline long to_p(JNIEnv* env, struct NValueInfo *nvinfo, void *e) {
   size_t i;
   struct transitem * ti;
-  if (NULL != nvinfo || NULL != nvinfo->transtable) {
+  if (NULL != nvinfo && NULL != nvinfo->transtable) {
     for (i = 0; i < nvinfo->transtablesz; ++i) {
       ti = nvinfo->transtable + i;
       if (e >= ti->base && e < ti->base + ti->size) {
@@ -302,7 +302,7 @@ inline long to_p(JNIEnv* env, struct NValueInfo *nvinfo, void *e) {
     }
     throw(env, "No item found in Translate Table.");
   } else {
-    return e;
+    return addr_to_java(e);
   }
   return -1L;
 }


Mime
View raw message