horn-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject incubator-horn git commit: Use hama-0.7.2-SNAPSHOT
Date Mon, 25 Apr 2016 02:31:06 GMT
Repository: incubator-horn
Updated Branches:
  refs/heads/master c0fea6b17 -> 5f001cf9a


Use hama-0.7.2-SNAPSHOT


Project: http://git-wip-us.apache.org/repos/asf/incubator-horn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-horn/commit/5f001cf9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-horn/tree/5f001cf9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-horn/diff/5f001cf9

Branch: refs/heads/master
Commit: 5f001cf9a77ffa0ac4dbcfcbf0db756f2458e115
Parents: c0fea6b
Author: Edward J. Yoon <edwardyoon@apache.org>
Authored: Mon Apr 25 09:39:34 2016 +0900
Committer: Edward J. Yoon <edwardyoon@apache.org>
Committed: Mon Apr 25 11:14:40 2016 +0900

----------------------------------------------------------------------
 pom.xml                                                |  4 ++--
 .../apache/horn/bsp/AbstractLayeredNeuralNetwork.java  |  2 +-
 src/main/java/org/apache/horn/bsp/AutoEncoder.java     |  2 +-
 src/main/java/org/apache/horn/bsp/HornJob.java         |  2 +-
 .../java/org/apache/horn/bsp/NeuralNetworkTrainer.java | 13 ++++++-------
 .../org/apache/horn/bsp/SmallLayeredNeuralNetwork.java |  8 +++++++-
 .../horn/bsp/SmallLayeredNeuralNetworkTrainer.java     |  1 -
 .../java/org/apache/horn/examples/NeuralNetwork.java   |  2 +-
 .../apache/horn/bsp/TestSmallLayeredNeuralNetwork.java |  3 +--
 .../org/apache/horn/examples/NeuralNetworkTest.java    |  1 -
 10 files changed, 20 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8e258d4..5f83793 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,8 +32,8 @@
 
   <properties>
     <commons-logging.version>1.1.1</commons-logging.version>
-    <hama.version>0.7.0</hama.version>
-    <hadoop.version>2.7.0</hadoop.version>
+    <hama.version>0.7.2-SNAPSHOT</hama.version>
+    <hadoop.version>2.7.2</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>
     <junit.version>4.8.1</junit.version>
     <log4j.version>1.2.16</log4j.version>

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java b/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java
index 1afe8f5..b18eb44 100644
--- a/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java
+++ b/src/main/java/org/apache/horn/bsp/AbstractLayeredNeuralNetwork.java
@@ -28,7 +28,7 @@ import org.apache.hama.commons.math.DoubleDoubleFunction;
 import org.apache.hama.commons.math.DoubleFunction;
 import org.apache.hama.commons.math.DoubleMatrix;
 import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
+import org.apache.horn.funcs.FunctionFactory;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/main/java/org/apache/horn/bsp/AutoEncoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/horn/bsp/AutoEncoder.java b/src/main/java/org/apache/horn/bsp/AutoEncoder.java
index a632942..8ea2930 100644
--- a/src/main/java/org/apache/horn/bsp/AutoEncoder.java
+++ b/src/main/java/org/apache/horn/bsp/AutoEncoder.java
@@ -27,8 +27,8 @@ import org.apache.hama.commons.math.DenseDoubleVector;
 import org.apache.hama.commons.math.DoubleFunction;
 import org.apache.hama.commons.math.DoubleMatrix;
 import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
 import org.apache.hama.ml.util.FeatureTransformer;
+import org.apache.horn.funcs.FunctionFactory;
 
 import com.google.common.base.Preconditions;
 

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/main/java/org/apache/horn/bsp/HornJob.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/horn/bsp/HornJob.java b/src/main/java/org/apache/horn/bsp/HornJob.java
index a9c7cc1..4521b87 100644
--- a/src/main/java/org/apache/horn/bsp/HornJob.java
+++ b/src/main/java/org/apache/horn/bsp/HornJob.java
@@ -22,7 +22,7 @@ import java.io.IOException;
 import org.apache.hama.HamaConfiguration;
 import org.apache.hama.bsp.BSPJob;
 import org.apache.hama.commons.math.Function;
-import org.apache.hama.commons.math.FunctionFactory;
+import org.apache.horn.funcs.FunctionFactory;
 
 public class HornJob extends BSPJob {
 

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java b/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java
index 1c8a198..648e86b 100644
--- a/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java
+++ b/src/main/java/org/apache/horn/bsp/NeuralNetworkTrainer.java
@@ -28,7 +28,6 @@ import org.apache.hama.bsp.BSP;
 import org.apache.hama.bsp.BSPPeer;
 import org.apache.hama.bsp.sync.SyncException;
 import org.apache.hama.commons.io.VectorWritable;
-import org.apache.hama.ml.perception.MLPMessage;
 import org.apache.hama.ml.util.DefaultFeatureTransformer;
 import org.apache.hama.ml.util.FeatureTransformer;
 
@@ -39,7 +38,7 @@ import org.apache.hama.ml.util.FeatureTransformer;
  * 
  */
 public abstract class NeuralNetworkTrainer extends
-    BSP<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> {
+    BSP<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> {
 
   protected static final Log LOG = LogFactory
       .getLog(NeuralNetworkTrainer.class);
@@ -53,7 +52,7 @@ public abstract class NeuralNetworkTrainer extends
   
   @Override
   final public void setup(
-      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage>
peer)
+      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
       throws IOException, SyncException, InterruptedException {
     conf = peer.getConfiguration();
     featureTransformer = new DefaultFeatureTransformer();
@@ -69,7 +68,7 @@ public abstract class NeuralNetworkTrainer extends
    * @throws InterruptedException
    */
   protected void extraSetup(
-      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage>
peer)
+      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
       throws IOException, SyncException, InterruptedException {
 
   }
@@ -79,12 +78,12 @@ public abstract class NeuralNetworkTrainer extends
    */
   @Override
   public abstract void bsp(
-      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage>
peer)
+      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
       throws IOException, SyncException, InterruptedException;
 
   @Override
   public void cleanup(
-      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage>
peer)
+      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
       throws IOException {
     this.extraCleanup(peer);
     // write model to modelPath
@@ -99,7 +98,7 @@ public abstract class NeuralNetworkTrainer extends
    * @throws InterruptedException
    */
   protected void extraCleanup(
-      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage>
peer)
+      BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, Synapse> peer)
       throws IOException {
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java
index aaad86e..0ea8e51 100644
--- a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java
+++ b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetwork.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hama.Constants;
 import org.apache.hama.HamaConfiguration;
 import org.apache.hama.bsp.BSPJob;
 import org.apache.hama.commons.io.MatrixWritable;
@@ -42,9 +43,9 @@ import org.apache.hama.commons.math.DenseDoubleVector;
 import org.apache.hama.commons.math.DoubleFunction;
 import org.apache.hama.commons.math.DoubleMatrix;
 import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
 import org.apache.hama.util.ReflectionUtils;
 import org.apache.horn.examples.MultiLayerPerceptron.StandardNeuron;
+import org.apache.horn.funcs.FunctionFactory;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
@@ -583,6 +584,11 @@ public class SmallLayeredNeuralNetwork extends AbstractLayeredNeuralNetwork
{
     job.getConfiguration().setClass("neuron.class", StandardNeuron.class,
         Neuron.class);
 
+    // additional for parameter server
+    // TODO at this moment, we use 1 task as a parameter server
+    // In the future, the number of parameter server should be configurable
+    job.getConfiguration().setInt(Constants.ADDITIONAL_BSP_TASKS, 1);
+
     job.setInputPath(new Path(conf.get("training.input.path")));
     job.setInputFormat(org.apache.hama.bsp.SequenceFileInputFormat.class);
     job.setInputKeyClass(LongWritable.class);

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java
index 58f96d1..c3e258c 100644
--- a/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java
+++ b/src/main/java/org/apache/horn/bsp/SmallLayeredNeuralNetworkTrainer.java
@@ -33,7 +33,6 @@ import org.apache.hama.commons.io.VectorWritable;
 import org.apache.hama.commons.math.DenseDoubleMatrix;
 import org.apache.hama.commons.math.DoubleMatrix;
 import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
 import org.apache.hama.ipc.RPC;
 
 import com.google.common.base.Preconditions;

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/main/java/org/apache/horn/examples/NeuralNetwork.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/horn/examples/NeuralNetwork.java b/src/main/java/org/apache/horn/examples/NeuralNetwork.java
index 1503ef0..5c0afdf 100644
--- a/src/main/java/org/apache/horn/examples/NeuralNetwork.java
+++ b/src/main/java/org/apache/horn/examples/NeuralNetwork.java
@@ -30,8 +30,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hama.HamaConfiguration;
 import org.apache.hama.commons.math.DenseDoubleVector;
 import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
 import org.apache.horn.bsp.SmallLayeredNeuralNetwork;
+import org.apache.horn.funcs.FunctionFactory;
 
 /**
  * The example of using {@link SmallLayeredNeuralNetwork}, including the

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java b/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java
index 972d55a..ee48136 100644
--- a/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java
+++ b/src/test/java/org/apache/horn/bsp/TestSmallLayeredNeuralNetwork.java
@@ -44,11 +44,11 @@ import org.apache.hama.commons.math.DenseDoubleMatrix;
 import org.apache.hama.commons.math.DenseDoubleVector;
 import org.apache.hama.commons.math.DoubleMatrix;
 import org.apache.hama.commons.math.DoubleVector;
-import org.apache.hama.commons.math.FunctionFactory;
 import org.apache.hama.ml.util.DefaultFeatureTransformer;
 import org.apache.hama.ml.util.FeatureTransformer;
 import org.apache.horn.bsp.AbstractLayeredNeuralNetwork.LearningStyle;
 import org.apache.horn.bsp.AbstractLayeredNeuralNetwork.TrainingMethod;
+import org.apache.horn.funcs.FunctionFactory;
 import org.junit.Test;
 import org.mortbay.log.Log;
 
@@ -616,7 +616,6 @@ public class TestSmallLayeredNeuralNetwork extends MLTestBase {
     trainingParameters.put("training.batch.size", "300");
     trainingParameters.put("convergence.check.interval", "1000");
     //ann.train(new HamaConfiguration(), tmpDatasetPath, trainingParameters);
-    
 
     long end = new Date().getTime();
 

http://git-wip-us.apache.org/repos/asf/incubator-horn/blob/5f001cf9/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java b/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java
index 5582dc0..932b17a 100644
--- a/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java
+++ b/src/test/java/org/apache/horn/examples/NeuralNetworkTest.java
@@ -60,7 +60,6 @@ public class NeuralNetworkTest extends HamaCluster {
         conf.get("bsp.master.address"));
     conf.set("bsp.local.dir", "/tmp/hama-test");
     conf.set(Constants.ZOOKEEPER_QUORUM, "localhost");
-    conf.setBoolean(Constants.FORCE_SET_BSP_TASKS, true);
     conf.setInt(Constants.ZOOKEEPER_CLIENT_PORT, 21810);
     conf.set("hama.sync.client.class",
         org.apache.hama.bsp.sync.ZooKeeperSyncClientImpl.class


Mime
View raw message