singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [2/2] incubator-singa git commit: SINGA-9 Add Support for Restricted Boltzman Machine (RBM) model
Date Fri, 28 Aug 2015 09:28:48 GMT
SINGA-9 Add Support for Restricted Boltzman Machine (RBM) model

* Refactor the CDWorker::TrainOneBatch.
* Replace Phase with int flag in ComputeFeature ComputeGradient
* functions. The flag can be a combination for multiple phases, e.g.,
* kTrain|kForward, where each phase is set to have only one none zero bit,
e.g., 1, 2, 4, 8, etc.
* Remove compliation dependency on opencv.
* Refactor JobProto to create a AlgProto for TrainOneBatch.
* Create a RBMLayer as the base layer for RBM layers.
* Updated all configurations of all examples.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/fbbcaafd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/fbbcaafd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/fbbcaafd

Branch: refs/heads/master
Commit: fbbcaafdba3e885eab44c0dcfd23829c2c80f732
Parents: ef4de79
Author: Wei Wang <wangwei@comp.nus.edu.sg>
Authored: Thu Aug 27 21:50:08 2015 +0800
Committer: wangwei <wangwei@comp.nus.edu.sg>
Committed: Fri Aug 28 17:18:38 2015 +0800

----------------------------------------------------------------------
 Makefile.am                     |   3 -
 configure.ac                    |  18 +-
 examples/cifar10/job.conf       |   5 +-
 examples/mnist/conv.conf        |   4 +-
 examples/mnist/job.conf         |  28 ++-
 examples/rbm/autoencoder.conf   |  89 ++--------
 examples/rbm/rbm0.conf          |  41 ++---
 examples/rbm/rbm1.conf          |  85 ++++-----
 examples/rbm/rbm2.conf          |  57 ++----
 examples/rbm/rbm3.conf          |  68 ++------
 include/mshadow/tensor_random.h |  17 +-
 include/neuralnet/base_layer.h  |  91 ++++++----
 include/neuralnet/layer.h       | 146 ++++------------
 include/trainer/worker.h        |   6 -
 include/utils/param.h           |   3 +-
 src/driver.cc                   |   8 +-
 src/neuralnet/base_layer.cc     |  49 +++---
 src/neuralnet/layer.cc          | 324 ++++++++++++++++-------------------
 src/proto/job.proto             |  47 +++--
 src/trainer/worker.cc           |  96 ++++-------
 src/utils/common.cc             |   2 +-
 src/utils/param.cc              |  10 +-
 22 files changed, 497 insertions(+), 700 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index fa28848..ae8a9dd 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -95,9 +95,6 @@ singa_LDFLAGS = -I./include \
                 -lglog  \
                 -lprotobuf \
                 -lrt \
-                -lopencv_highgui \
-                -lopencv_imgproc \
-                -lopencv_core \
                 -lopenblas \
                 -lzmq \
                 -lczmq \

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 35c6d61..569c406 100644
--- a/configure.ac
+++ b/configure.ac
@@ -44,15 +44,15 @@ if test x"$enable_lmdb" = x"yes"; then
 	AC_DEFINE(LMDB, 1, [Enable Option layer])
 fi
 
-AC_CHECK_LIB([opencv_imgproc], [main], [], [
-  AC_MSG_ERROR([unable to find opencv_imgproc lib])
-  ])
-AC_CHECK_LIB([opencv_highgui], [main], [], [
-  AC_MSG_ERROR([unable to find opencv_highgui lib])
-  ])
-AC_CHECK_LIB([opencv_core], [main], [], [
-  AC_MSG_ERROR([unable to find opencv_core lib])
-  ])
+#AC_CHECK_LIB([opencv_imgproc], [main], [], [
+#  AC_MSG_ERROR([unable to find opencv_imgproc lib])
+#  ])
+#AC_CHECK_LIB([opencv_highgui], [main], [], [
+#  AC_MSG_ERROR([unable to find opencv_highgui lib])
+#  ])
+#AC_CHECK_LIB([opencv_core], [main], [], [
+#  AC_MSG_ERROR([unable to find opencv_core lib])
+#  ])
 AC_CHECK_LIB([zookeeper_mt], [main], [], [
   AC_MSG_ERROR([unable to find zookeeper])
   ])

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index f44ca50..9d25904 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -3,7 +3,10 @@ train_steps: 1000
 test_steps: 100
 test_freq:300
 disp_freq:30
-alg: kBP
+debug: true
+train_one_batch {
+  alg: kBP
+}
 updater{
   type: kSGD
   weight_decay:0.004

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/mnist/conv.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/conv.conf b/examples/mnist/conv.conf
index 1d4d740..aaf34f2 100644
--- a/examples/mnist/conv.conf
+++ b/examples/mnist/conv.conf
@@ -3,7 +3,9 @@ train_steps: 10000
 test_steps:100
 test_freq:500
 disp_freq:50
-alg: kBP
+train_one_batch {
+  alg: kBP
+}
 updater {
   momentum:0.9
   weight_decay:0.0005

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/mnist/job.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/job.conf b/examples/mnist/job.conf
index 360e1ec..b8d14e8 100644
--- a/examples/mnist/job.conf
+++ b/examples/mnist/job.conf
@@ -3,7 +3,9 @@ train_steps: 1000
 test_steps:10
 test_freq:60
 disp_freq:10
-alg: kBP
+train_one_batch {
+  alg: kBP
+}
 updater{
   type: kSGD
   learning_rate{
@@ -82,6 +84,10 @@ neuralnet {
   layer{
     name: "tanh1"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
     srclayers:"fc1"
   }
   layer{
@@ -112,6 +118,11 @@ neuralnet {
   layer{
     name: "tanh2"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc2"
   }
   layer{
@@ -143,6 +154,11 @@ neuralnet {
   layer{
     name: "tanh3"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc3"
   }
   layer{
@@ -174,6 +190,11 @@ neuralnet {
   layer{
     name: "tanh4"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc4"
   }
   layer{
@@ -205,6 +226,11 @@ neuralnet {
   layer{
     name: "tanh5"
     type: kTanh
+    tanh_conf {
+      outer_scale: 1.7159047
+      inner_scale: 0.6666667
+    }
+
     srclayers:"fc5"
   }
   layer{

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/rbm/autoencoder.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/autoencoder.conf b/examples/rbm/autoencoder.conf
index 9575323..bc32cc7 100644
--- a/examples/rbm/autoencoder.conf
+++ b/examples/rbm/autoencoder.conf
@@ -1,15 +1,15 @@
-name: "deep-big-simple-mlp"
+name: "auto-encoder"
 train_steps: 12200
 test_steps:100
-test_freq:100
-disp_freq:20
-checkpoint_after: 1000
-checkpoint_freq: 1000
-checkpoint_path: "examples/rbm/checkpoint/rbm0/checkpoint/step6000-worker0.bin"
-checkpoint_path: "examples/rbm/checkpoint/rbm1/checkpoint/step6000-worker0.bin"
-checkpoint_path: "examples/rbm/checkpoint/rbm2/checkpoint/step6000-worker0.bin"
-checkpoint_path: "examples/rbm/checkpoint/rbm3/checkpoint/step6000-worker0.bin"
-alg: kBP
+test_freq:1000
+disp_freq:100
+checkpoint_path: "examples/rbm/rbm0/checkpoint/step6000-worker0.bin"
+checkpoint_path: "examples/rbm/rbm1/checkpoint/step6000-worker0.bin"
+checkpoint_path: "examples/rbm/rbm2/checkpoint/step6000-worker0.bin"
+checkpoint_path: "examples/rbm/rbm3/checkpoint/step6000-worker0.bin"
+train_one_batch{
+  alg: kBP
+}
 updater{
   type: kAdaGrad
   learning_rate{
@@ -23,7 +23,7 @@ neuralnet {
     name: "data"
     type: kShardData
     sharddata_conf {
-      path: "examples/rbm/mnist_train_shard"
+      path: "examples/mnist/mnist_train_shard"
       batchsize: 1000
     }
     exclude: kTest
@@ -33,7 +33,7 @@ neuralnet {
     name: "data"
     type: kShardData
     sharddata_conf {
-      path: "examples/rbm/mnist_test_shard"
+      path: "examples/mnist/mnist_test_shard"
       batchsize: 1000
     }
     exclude: kTrain
@@ -64,19 +64,9 @@ neuralnet {
     }
     param{
       name: "w1"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb12"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -94,19 +84,9 @@ neuralnet {
     }
     param{
       name: "w2"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb22"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -125,19 +105,9 @@ neuralnet {
     }
     param{
       name: "w3"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb32"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -156,19 +126,10 @@ neuralnet {
     }
     param{
       name: "w4"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb42"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
+
     }
   }
 
@@ -187,11 +148,6 @@ neuralnet {
     }
     param{
       name: "rb41"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -214,13 +170,7 @@ neuralnet {
     }
     param{
       name: "rb31"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
-
   }
 
   layer{
@@ -242,11 +192,6 @@ neuralnet {
     }
     param{
       name: "rb21"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
 
   }
@@ -270,13 +215,7 @@ neuralnet {
     }
     param{
       name: "rb11"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
-
   }
 
   layer{
@@ -295,5 +234,5 @@ neuralnet {
 cluster {
   nworker_groups: 1
   nserver_groups: 1
-  workspace: "examples/rbm/checkpoint/autoencoder/"
+  workspace: "examples/rbm/autoencoder/"
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/rbm/rbm0.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm0.conf b/examples/rbm/rbm0.conf
index ef8653f..a7e503b 100644
--- a/examples/rbm/rbm0.conf
+++ b/examples/rbm/rbm0.conf
@@ -1,11 +1,11 @@
-name: "deep-big-simple-dbm"
+name: "rbm0"
 train_steps: 6000
 test_steps:100
 test_freq:100
 disp_freq: 100
-alg: kCD
-checkpoint_after: 500
-checkpoint_freq: 1000
+train_one_batch{
+  alg: kCD
+}
 updater{
   type: kSGD
   momentum: 0.9
@@ -21,7 +21,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_train_shard"
+    path: "examples/mnist/mnist_train_shard"
     batchsize: 100
   }
   exclude: kTest
@@ -32,7 +32,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_test_shard"
+    path: "examples/mnist/mnist_test_shard"
     batchsize: 100
   }
   exclude: kTrain
@@ -54,22 +54,15 @@ layer{
   type: kRBMVis
   srclayers:"mnist"
   srclayers:"RBMHid"
-  rbmvis_conf{
-    num_output: 1000
-  }
   param{
-    name: "w1"
-    init{
-    type: kGaussian
-    mean: 0.0
-    std: 0.1
-    }
+    name: "w1_"
+    share_from: "w1"
   }
   param{
     name: "rb11"
     init{
-    type: kConstant
-    value: 0.0
+      type: kConstant
+      value: 0.0
     }
   }
 }
@@ -82,14 +75,18 @@ layer{
     hid_dim: 1000
   }
   param{
-    name: "w1_1"
-    share_from: "w1"
+    name: "w1"
+    init{
+      type: kGaussian
+      mean: 0.0
+      std: 0.1
+    }
   }
   param{
     name: "rb12"
     init{
-    type: kConstant
-    value: 0.0
+      type: kConstant
+      value: 0.0
     }
   }
 }
@@ -99,5 +96,5 @@ cluster {
   nserver_groups: 1
   nservers_per_group: 1
   nworkers_per_group: 1
-  workspace: "examples/rbm/checkpoint/rbm0/"
+  workspace: "examples/rbm/rbm0/"
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/rbm/rbm1.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm1.conf b/examples/rbm/rbm1.conf
index f9b4974..db27d3a 100644
--- a/examples/rbm/rbm1.conf
+++ b/examples/rbm/rbm1.conf
@@ -1,12 +1,12 @@
-name: "deep-big-simple-dbm"
+name: "rbm1"
 train_steps: 6000
 test_steps:100
-test_freq:500
+test_freq:1000
 disp_freq: 100
-alg: kCD
-checkpoint_after: 500
-checkpoint_freq: 1000
-checkpoint_path: "examples/rbm/checkpoint/rbm0/checkpoint/step6000-worker0.bin"
+train_one_batch{
+  alg: kCD
+}
+checkpoint_path: "examples/rbm/rbm0/checkpoint/step6000-worker0.bin"
 updater{
   type: kSGD
   momentum: 0.9
@@ -22,7 +22,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_train_shard"
+    path: "examples/mnist/mnist_train_shard"
     batchsize: 100
   }
   exclude: kTest
@@ -33,7 +33,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_test_shard"
+    path: "examples/mnist/mnist_test_shard"
     batchsize: 100
   }
   exclude: kTrain
@@ -51,51 +51,34 @@ layer{
 }
 
 layer{
-    name: "fc1"
-    type: kInnerProduct
-    srclayers:"mnist"
-    innerproduct_conf{
-      num_output: 1000
-    }
-    param{
-      name: "w1"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
-    }
-    param{
-      name: "rb12"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
-    }
+  name: "fc1"
+  type: kInnerProduct
+  srclayers:"mnist"
+  innerproduct_conf{
+    num_output: 1000
   }
-
-  layer{
-    name: "sigmoid1"
-    type: kSigmoid
-    srclayers:"fc1"
+  param{
+    name: "w1"
   }
+  param{
+    name: "rb12"
+  }
+}
+
+layer{
+  name: "sigmoid1"
+  type: kSigmoid
+  srclayers:"fc1"
+}
 
 layer{
   name: "RBMVis"
   type: kRBMVis
   srclayers:"sigmoid1"
   srclayers:"RBMHid"
-  rbmvis_conf{
-    num_output: 500
-  }
   param{
-    name: "w2"
-    init{
-    type: kGaussian
-    mean: 0.0
-    std: 0.1
-    }
+    name: "w2_"
+    share_from: "w2"
   }
   param{
     name: "rb21"
@@ -114,14 +97,18 @@ layer{
     hid_dim: 500
   }
   param{
-    name: "w2_1"
-    share_from: "w2"
+    name: "w2"
+    init{
+      type: kGaussian
+      mean: 0.0
+      std: 0.1
+    }
   }
   param{
     name: "rb22"
     init{
-    type: kConstant
-    value: 0.0
+      type: kConstant
+      value: 0.0
     }
   }
 }
@@ -131,5 +118,5 @@ cluster {
   nserver_groups: 1
   nservers_per_group: 1
   nworkers_per_group: 1
-  workspace: "examples/rbm/checkpoint/rbm1/"
+  workspace: "examples/rbm/rbm1/"
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/rbm/rbm2.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm2.conf b/examples/rbm/rbm2.conf
index 6629481..fd08907 100644
--- a/examples/rbm/rbm2.conf
+++ b/examples/rbm/rbm2.conf
@@ -1,12 +1,12 @@
-name: "deep-big-simple-dbm"
+name: "rbm2"
 train_steps: 6000
 test_steps:100
-test_freq:100
+test_freq:1000
 disp_freq: 100
-alg: kCD
-checkpoint_after: 500
-checkpoint_freq: 1000
-checkpoint_path: "examples/rbm/checkpoint/rbm1/checkpoint/step6000-worker0.bin"
+train_one_batch{
+  alg: kCD
+}
+checkpoint_path: "examples/rbm/rbm1/checkpoint/step6000-worker0.bin"
 
 updater{
   type: kSGD
@@ -24,7 +24,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_train_shard"
+    path: "examples/mnist/mnist_train_shard"
     batchsize: 100
   }
   exclude: kTest
@@ -35,7 +35,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_test_shard"
+    path: "examples/mnist/mnist_test_shard"
     batchsize: 100
   }
   exclude: kTrain
@@ -61,19 +61,9 @@ layer{
     }
     param{
       name: "w1"
-      init {
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb12"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -92,19 +82,9 @@ layer{
     }
     param{
       name: "w2"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb22"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -118,16 +98,9 @@ layer{
   type: kRBMVis
   srclayers:"sigmoid2"
   srclayers:"RBMHid"
-  rbmvis_conf{
-    num_output: 250
-  }
   param{
-    name: "w3"
-    init{
-    type: kGaussian
-    mean: 0.0
-    std: 0.1
-    }
+    name: "w3_"
+    share_from: "w3"
   }
   param{
     name: "rb31"
@@ -146,8 +119,12 @@ layer{
     hid_dim: 250
   }
   param{
-    name: "w3_1"
-    share_from: "w3"
+    name: "w3"
+    init{
+      type: kGaussian
+      mean: 0.0
+      std: 0.1
+    }
   }
   param{
     name: "rb32"
@@ -163,5 +140,5 @@ cluster {
   nserver_groups: 1
   nservers_per_group: 1
   nworkers_per_group: 1
-  workspace: "examples/rbm/checkpoint/rbm2/"
+  workspace: "examples/rbm/rbm2/"
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/examples/rbm/rbm3.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm3.conf b/examples/rbm/rbm3.conf
index 482c5e7..fe7cc1f 100644
--- a/examples/rbm/rbm3.conf
+++ b/examples/rbm/rbm3.conf
@@ -1,12 +1,12 @@
-name: "deep-big-simple-dbm"
+name: "rbm3"
 train_steps: 6000
 test_steps: 100
-test_freq: 100
+test_freq: 1000
 disp_freq: 100
-alg: kCD
-checkpoint_after: 500
-checkpoint_freq: 1000
-checkpoint_path: "examples/rbm/checkpoint/rbm2/checkpoint/step6000-worker0.bin"
+train_one_batch{
+  alg: kCD
+}
+checkpoint_path: "examples/rbm/rbm2/checkpoint/step6000-worker0.bin"
 updater{
     type: kSGD
     momentum: 0.9
@@ -22,7 +22,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_train_shard"
+    path: "examples/mnist/mnist_train_shard"
     batchsize: 100
   }
   exclude: kTest
@@ -33,7 +33,7 @@ layer {
   name: "data"
   type: kShardData
   sharddata_conf {
-    path: "examples/rbm/mnist_test_shard"
+    path: "examples/mnist/mnist_test_shard"
     batchsize: 100
   }
   exclude: kTrain
@@ -59,19 +59,9 @@ layer{
     }
     param{
       name: "w1"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb12"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -90,19 +80,9 @@ layer{
     }
     param{
       name: "w2"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb22"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -121,19 +101,9 @@ layer{
     }
     param{
       name: "w3"
-      init{
-      type: kUniform
-      low:-0.05
-      high:0.05
-      }
     }
     param{
       name: "rb32"
-      init{
-      type: kUniform
-      low: -0.05
-      high:0.05
-      }
     }
   }
 
@@ -148,16 +118,10 @@ layer{
   type: kRBMVis
   srclayers:"sigmoid3"
   srclayers:"RBMHid"
-  rbmvis_conf{
-    num_output: 30
-  }
   param{
-    name: "w4"
-    init{
-    type: kGaussian
-    mean: 0.0
-    std: 0.1
-    }
+    name: "w4_"
+    share_from: "w4"
+
   }
   param{
     name: "rb41"
@@ -177,8 +141,12 @@ layer{
     gaussian: true
   }
   param{
-    name: "w4_1"
-    share_from: "w4"
+    name: "w4"
+    init{
+      type: kGaussian
+      mean: 0.0
+      std: 0.1
+    }
   }
   param{
     name: "rb42"
@@ -194,5 +162,5 @@ cluster {
   nserver_groups: 1
   nservers_per_group: 1
   nworkers_per_group: 1
-  workspace: "examples/rbm/checkpoint/rbm3/"
+  workspace: "examples/rbm/rbm3/"
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/include/mshadow/tensor_random.h
----------------------------------------------------------------------
diff --git a/include/mshadow/tensor_random.h b/include/mshadow/tensor_random.h
index 72164a8..59ef082 100644
--- a/include/mshadow/tensor_random.h
+++ b/include/mshadow/tensor_random.h
@@ -68,20 +68,27 @@ namespace mshadow {
             gen_.seed(seed);
             #endif
         }
+        template<int dim>
+        inline void SampleBinary(Tensor<cpu, dim> &src) {
+          SampleBinary(src, src);
+        }
+
         /*!
          * \brief generate binary data according to a probability matrix
+         * \param src source
          * \param dst destination
          * \param a lower bound of uniform
          * \param b upper bound of uniform
          * \tparam dim dimension of tensor
          */
         template<int dim>
-        inline void SampleBinary( Tensor<cpu, dim> &dst) {
+        inline void SampleBinary(Tensor<cpu, dim> &dst, Tensor<cpu, dim> &src) {
             real_t a=0.0f;
             real_t b=1.0f;
-            Tensor<cpu, 2> mat = dst.FlatTo2D();
+            Tensor<cpu, 2> dmat = dst.FlatTo2D();
+            Tensor<cpu, 2> smat = src.FlatTo2D();
             std::uniform_real_distribution<real_t> distribution (a,b);
-            for ( index_t i = 0; i < mat.shape[1]; ++i ) {
+            for ( index_t i = 0; i < dmat.shape[1]; ++i ) {
                 #if MSHADOW_USE_MKL
                 #if MSHADOW_SINGLE_PRECISION
                 int status = vsRngUniform( 0, vStream_, mat.shape[0], mat[i].dptr, a, b );
@@ -96,8 +103,8 @@ namespace mshadow {
                     mat[i][j] = this->RandNext()*(b-a) + a;
                 }
                 */
-                for ( index_t j = 0; j < mat.shape[0]; ++j ) {
-                    mat[i][j] = distribution(gen_) > mat[i][j] ? 0.0f: 1.0f;
+                for ( index_t j = 0; j < dmat.shape[0]; ++j ) {
+                    dmat[i][j] = distribution(gen_) > smat[i][j] ? 0.0f: 1.0f;
                 }
                 #endif
             }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/include/neuralnet/base_layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/base_layer.h b/include/neuralnet/base_layer.h
index 5575fc7..9aa207d 100644
--- a/include/neuralnet/base_layer.h
+++ b/include/neuralnet/base_layer.h
@@ -49,25 +49,25 @@ class Layer {
   /**
    * Compute features of this layer based on connected layers.
    *
-   * @param phase kTrain, kTest, kPositive, etc.
+   * @param flag kTrain, kTest, kPositive, etc.
    */
-  virtual void ComputeFeature(Phase phase, Metric* perf) = 0;
+  virtual void ComputeFeature(int flag, Metric* perf) = 0;
   /**
    * Compute gradients for parameters and connected layers.
    *
-   * @param phase kTrain, kTest, kPositive, etc.
+   * @param flag kTrain, kTest, kPositive, etc.
    */
   virtual void ComputeLoss(Metric* perf) {}
-  virtual void ComputeGradient(Phase phase) = 0;
+  virtual void ComputeGradient(int flag) = 0;
   /**
    * For print debug info about each layer, e.g., norm of feature vector,
    * norm of parameters.
    *
    * @param step training/test/validation step
-   * @param phase forward/backward/positive/negative...
+   * @param flag forward/backward/positive/negative...
    * @return debug info about this layer.
    */
-  const string DebugString(int step, Phase phase);
+  const string DebugString(int step, int flag);
   /**
    * Layers that have paramters must override this function.
    *
@@ -141,10 +141,10 @@ class Layer {
   /**
    * @return a const ref for Blob storing neuron values of this layer for BP
    */
-  virtual const Blob<float>& data(const Layer* from, Phase = kPositive) const {
+  virtual const Blob<float>& data(const Layer* from) const {
     return data_;
   }
-  virtual Blob<float>* mutable_data(const Layer* from, Phase = kPositive) {
+  virtual Blob<float>* mutable_data(const Layer* from) {
     return &data_;
   }
 
@@ -246,15 +246,15 @@ class BridgeSrcLayer: public BridgeLayer {
   using Layer::ComputeFeature;
   using Layer::ComputeGradient;
 
-  void ComputeFeature(Phase phase, Metric* perf) override {}
-  void ComputeGradient(Phase phase) override {
+  void ComputeFeature(int flag, Metric* perf) override {}
+  void ComputeGradient(int flag) override {
     ready_ = false;
   }
 
-  const Blob<float>& data(const Layer* from, Phase phase) const override {
+  const Blob<float>& data(const Layer* from) const override {
     return srclayers_[0]->data(this);
   }
-  Blob<float>* mutable_data(const Layer* from, Phase phase) override {
+  Blob<float>* mutable_data(const Layer* from) override {
     return srclayers_[0]->mutable_data(this);
   }
   const Blob<float>& grad(const Layer* from) const override {
@@ -278,11 +278,11 @@ class BridgeDstLayer: public BridgeLayer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric* perf) override {
+  void ComputeFeature(int flag, Metric* perf) override {
     // reset ready_ for next iteration.
     ready_ = false;
   }
-  void ComputeGradient(Phase phase) override {}
+  void ComputeGradient(int flag) override {}
   bool is_bridgedstlayer() const {
     return true;
   }
@@ -297,8 +297,8 @@ class ConcateLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric* perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric* perf) override;
+  void ComputeGradient(int flag) override;
 };
 
 /**
@@ -311,11 +311,11 @@ class DataLayer: public Layer{
   using Layer::mutable_grad;
   using Layer::dst_layer_connection;
 
-  void ComputeGradient(Phase phase) override {}
+  void ComputeGradient(int flag) override {}
   bool is_datalayer() const override {
     return true;
   }
-  Blob<float>* mutable_data(const Layer* layer, Phase phase) override {
+  Blob<float>* mutable_data(const Layer* layer) override {
     return nullptr;
   }
   Blob<float>* mutable_grad(const Layer* layer) override {
@@ -357,11 +357,11 @@ class PrefetchLayer : public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric* perf) override;
-  void ComputeGradient(Phase phase) override {};
+  void ComputeFeature(int flag, Metric* perf) override;
+  void ComputeGradient(int flag) override {};
 
-  const Blob<float>& data(const Layer* from, Phase phase) const override;
-  Blob<float>* mutable_data(const Layer* layer, Phase phase) override;
+  const Blob<float>& data(const Layer* from) const override;
+  Blob<float>* mutable_data(const Layer* layer) override;
 
   Blob<float>* mutable_grad(const Layer* layer) override {
     return nullptr;
@@ -371,7 +371,7 @@ class PrefetchLayer : public Layer {
     return grad_;
   }
 
-  void Prefetch(Phase phase);
+  void Prefetch(int flag);
   virtual ~PrefetchLayer();
 
  protected:
@@ -389,14 +389,14 @@ class SliceLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric* perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric* perf) override;
+  void ComputeGradient(int flag) override;
   ConnectionType dst_layer_connection() const override {
     return kOneToMany;
   }
-  const Blob<float>& data(const Layer* layer, Phase phase) const override;
+  const Blob<float>& data(const Layer* layer) const override;
   const Blob<float>& grad(const Layer* layer) const override;
-  Blob<float>* mutable_data(const Layer* layer, Phase phase) override;
+  Blob<float>* mutable_data(const Layer* layer) override;
   Blob<float>* mutable_grad(const Layer* layer) override;
 
  protected:
@@ -418,8 +418,8 @@ class SplitLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric* perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric* perf) override;
+  void ComputeGradient(int flag) override;
   ConnectionType dst_layer_connection() const override {
     return kOneToMany;
   }
@@ -462,12 +462,12 @@ class ParserLayer: public Layer {
   using Layer::mutable_grad;
   using Layer::grad;
 
-  void ComputeFeature(Phase phase, Metric* perf) override;
-  void ComputeGradient(Phase phase) override {};
+  void ComputeFeature(int flag, Metric* perf) override;
+  void ComputeGradient(int flag) override {};
   /**
    * Parse records from DataLayer into blob.
    */
-  virtual void ParseRecords(Phase phase, const vector<Record>& records,
+  virtual void ParseRecords(int flag, const vector<Record>& records,
       Blob<float>* blob) = 0;
   bool is_parserlayer() const override {
     return true;
@@ -480,6 +480,33 @@ class ParserLayer: public Layer {
     return grad_;
   }
 };
+
+class RBMLayer: public Layer {
+ public:
+  const Blob<float>& neg_data(const Layer* layer) {
+    return neg_data_;
+  }
+  Blob<float>* mutable_neg_data(const Layer* layer) {
+    return &neg_data_;
+  }
+  const vector<Param*> GetParams() const override {
+    vector<Param*> params{weight_, bias_};
+    return params;
+  }
+  virtual Blob<float>* Sample(int flat) = 0;
+
+ protected:
+  //! dimension of the hidden layer
+  int hdim_;
+  //! dimension of the visible layer
+  int vdim_;
+  int batchsize_;
+  Param* weight_, *bias_;
+
+  Blob<float> neg_data_;
+  Blob<float> neg_sample_;
+  Blob<float> sample_;
+};
 }  // namespace singa
 
 #endif  // SINGA_NEURALNET_BASE_LAYER_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/include/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/layer.h b/include/neuralnet/layer.h
index b1fbbb0..435d854 100644
--- a/include/neuralnet/layer.h
+++ b/include/neuralnet/layer.h
@@ -31,8 +31,8 @@ class ConvolutionLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
   const vector<Param*> GetParams() const override {
     vector<Param*> params{weight_, bias_};
     return params;
@@ -57,8 +57,8 @@ class DropoutLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 
  protected:
   // drop probability
@@ -68,112 +68,42 @@ class DropoutLayer: public Layer {
    */
   Blob<float> mask_;
 };
+
 /**
  * RBM visible layer
  */
-class RBMVisLayer: public Layer {
+class RBMVisLayer: public RBMLayer {
  public:
   using Layer::ComputeFeature;
   using Layer::ComputeGradient;
 
-  void Setup(const LayerProto& proto,
-      int npartitions) override;
-  virtual bool is_vislayer() const {
-    return true;
-  }
-
-  void ComputeFeature(Phase phase,
-     Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
-  virtual void ComputeLoss(Metric* perf);
-  virtual Blob<float>* mutable_data(const Layer* from, Phase phase) {
-    if (phase == kPositive) {
-      return &data_;
-    } else {
-       return &vis_sample_;
-    }
-  }
-  virtual const Blob<float>& data(const Layer* from, Phase phase) const {
-    if (phase == kPositive) {
-      return data_;
-    } else {
-       return vis_sample_;
-    }
-  }
-  // virtual void ToProto(LayerProto *layer_proto, bool copyData);
-  const vector<Param*> GetParams() const override {
-    vector<Param*> params{weight_, bias_};
-    return params;
-  }
   ~RBMVisLayer();
-
-
+  void Setup(const LayerProto& proto, int npartitions) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
+  Blob<float>* Sample(int flat) override;
 
  private:
-  //! dimension of the hidden layer
-  int hdim_;
-  //! dimension of the visible layer
-  int vdim_;
-  int batchsize_;
-  // batchsize of negative phase
-  int neg_batchsize_;
-  bool is_first_iteration_vis_;
-  float scale_;
-  // srclayer index
-  int data_idx_;
-  int hid_idx_;
-  Param* weight_, *bias_;
-  // data to store sampling result
-  Blob<float> vis_sample_;
-  // in order to implement Persistent Contrastive Divergence,
+  RBMLayer* hid_layer_;
+  Layer* input_layer_;
 };
 /**
  * RBM hidden layer
  */
-class RBMHidLayer: public Layer {
+class RBMHidLayer: public RBMLayer {
  public:
   using Layer::ComputeFeature;
   using Layer::ComputeGradient;
 
-  void Setup(const LayerProto& proto,
-      int npartitions) override;
-  virtual bool is_hidlayer() const {
-    return true;
-  }
-
-  void ComputeFeature(Phase phase,
-     Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
-  virtual Blob<float>* mutable_data(const Layer* from, Phase phase) {
-    if (phase == kPositive)
-      return &data_;
-    else
-      return &hid_sample_;
-  }
-  virtual const Blob<float>& data(const Layer* from, Phase phase) const {
-    if (phase == kPositive)
-      return data_;
-    else
-      return hid_sample_;
-    }
-  const vector<Param*> GetParams() const override {
-    vector<Param*> params{weight_, bias_};
-    return params;
-  }
   ~RBMHidLayer();
-
+  void Setup(const LayerProto& proto, int npartitions) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
+  Blob<float>* Sample(int flat) override;
  private:
-  //! dimension of the hidden layer
-  int hdim_;
-  int vdim_;  // dimension of visible layer
-  int batchsize_;
-  // batchsize of negative phase
-  int neg_batchsize_;
-  float scale_;
   // whether use gaussian sampling
   bool gaussian_;
-  Blob<float> hid_sample_;
-  Param* weight_, *bias_;
+  RBMLayer *vis_layer_;
 };
 /**
   * fully connected layer
@@ -184,8 +114,8 @@ class InnerProductLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 
   ConnectionType src_neuron_connection(int k) const override {
     // CHECK_LT(k, srclayers_.size());
@@ -212,7 +142,7 @@ class LabelLayer: public ParserLayer {
   using ParserLayer::ParseRecords;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ParseRecords(Phase phase, const vector<Record>& records,
+  void ParseRecords(int flag, const vector<Record>& records,
       Blob<float>* blob) override;
 };
 
@@ -229,8 +159,8 @@ class LRNLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 
  protected:
   //! shape of the bottom layer feature
@@ -247,7 +177,7 @@ class MnistLayer: public ParserLayer {
   using ParserLayer::ParseRecords;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ParseRecords(Phase phase, const vector<Record>& records,
+  void ParseRecords(int flag, const vector<Record>& records,
       Blob<float>* blob) override;
   ConnectionType dst_layer_connection() const override {
     return kOneToMany;
@@ -269,8 +199,8 @@ class PoolingLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 
  protected:
   int kernel_, pad_, stride_;
@@ -284,8 +214,8 @@ class ReLULayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions = 1) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 };
 
 class EuclideanLossLayer: public LossLayer {
@@ -294,8 +224,8 @@ class EuclideanLossLayer: public LossLayer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 
 
   int partition_dim() const override {
@@ -321,8 +251,8 @@ class SoftmaxLossLayer: public LossLayer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 
   /**
    * softmax is not recommendeded for partition because it requires the whole
@@ -349,7 +279,7 @@ class RGBImageLayer: public ParserLayer {
   using ParserLayer::ParseRecords;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ParseRecords(Phase phase, const vector<Record>& records,
+  void ParseRecords(int flag, const vector<Record>& records,
       Blob<float>* blob) override;
 
  private:
@@ -365,7 +295,7 @@ class ShardDataLayer: public DataLayer{
 
   ~ShardDataLayer();
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
+  void ComputeFeature(int flag, Metric *perf) override;
 
  private:
   DataShard* shard_;
@@ -382,8 +312,8 @@ class SigmoidLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 };
 
 /**
@@ -397,8 +327,8 @@ class TanhLayer: public Layer {
   using Layer::ComputeGradient;
 
   void Setup(const LayerProto& proto, int npartitions) override;
-  void ComputeFeature(Phase phase, Metric *perf) override;
-  void ComputeGradient(Phase phase) override;
+  void ComputeFeature(int flag, Metric *perf) override;
+  void ComputeGradient(int flag) override;
 
  private:
   float outer_scale_, inner_scale_;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/include/trainer/worker.h
----------------------------------------------------------------------
diff --git a/include/trainer/worker.h b/include/trainer/worker.h
index 86b1c90..35ce77e 100644
--- a/include/trainer/worker.h
+++ b/include/trainer/worker.h
@@ -192,15 +192,9 @@ class BPWorker: public Worker{
 
 class CDWorker: public Worker{
  public:
-  ~CDWorker() {}
-  void Init(int thread_id, int grp_id, int id) override;
   void TrainOneBatch(int step, Metric* perf) override;
   void TestOneBatch(int step, Phase phase, shared_ptr<NeuralNet> net,
       Metric* perf) override;
-  void PositivePhase(int step, shared_ptr<NeuralNet> net, Metric* perf);
-  void NegativePhase(int step, shared_ptr<NeuralNet> net, Metric* perf);
-  void GradientPhase(int step, shared_ptr<NeuralNet> net);
-  void LossPhase(int step, shared_ptr<NeuralNet> net, Metric* perf);
 };
 
 inline int BlobTrgt(int grp, int layer) {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/include/utils/param.h
----------------------------------------------------------------------
diff --git a/include/utils/param.h b/include/utils/param.h
index f7a0982..0d24e95 100644
--- a/include/utils/param.h
+++ b/include/utils/param.h
@@ -74,13 +74,14 @@ class Param {
   static Param* Create(const ParamProto& proto);
   Param();
   virtual ~Param() {}
+  void Init(const ParamProto& proto) { proto_ = proto; }
   /**
    * Setup param object
    *
    * @param conf param configuration, include learning rate multiplier etc.
    * @param shape one value per dimension
    */
-  virtual void Setup(const ParamProto& conf, const std::vector<int>& shape);
+  virtual void Setup(const std::vector<int>& shape);
   /*
    * Fill the values according to init method, e.g., gaussian distribution.
    *

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index e5045a3..9fa4b86 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -1,9 +1,12 @@
-#include "singa.h"
 
 #include <cblas.h>
 #include <glog/logging.h>
 #include <string>
 
+#include "singa.h"
+
+#include "utils/tinydir.h"
+
 namespace singa {
 
 void Driver::Init(int argc, char **argv) {
@@ -89,6 +92,9 @@ void Driver::Submit(bool resume, const JobProto& jobConf) {
   if (singa_conf_.has_log_dir())
     SetupLog(singa_conf_.log_dir(), std::to_string(job_id_)
              + "-" + jobConf.name());
+  tinydir_dir workspace;
+  if (tinydir_open(&workspace, jobConf.cluster().workspace().c_str()) == -1)
+    LOG(FATAL) << "workspace does not exist: " << jobConf.cluster().workspace();
   if (jobConf.num_openblas_threads() != 1)
     LOG(WARNING) << "openblas with "
       << jobConf.num_openblas_threads() << " threads";

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/src/neuralnet/base_layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/base_layer.cc b/src/neuralnet/base_layer.cc
index f995353..46f8b57 100644
--- a/src/neuralnet/base_layer.cc
+++ b/src/neuralnet/base_layer.cc
@@ -1,5 +1,3 @@
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
 #include <cblas.h>
 #include <math.h>
 #include <cfloat>
@@ -24,14 +22,13 @@ void Layer::Setup(const LayerProto& proto, int npartitions) {
   layer_proto_ = proto;
 }
 
-const string Layer::DebugString(int step, Phase phase) {
+const string Layer::DebugString(int step, int flag) {
   string ret =StringPrintf("Layer %10s ", name().c_str());
-  if(data_.count() != 0)
-    return ret;
-  if(phase == kForward) {
-    ret += StringPrintf("data %10s data norm1 %13.9f", data_.asum_data());
-  }else if(phase == kBackward) {
-    ret += StringPrintf("grad norm1 %13.9f\n", grad_.asum_data());
+  if ((flag & kForward) == kForward && data_.count() !=0) {
+    ret += StringPrintf("data norm1 %13.9f", data_.asum_data());
+  } else if ((flag & kBackward) == kBackward) {
+    if (grad_.count() != 0)
+      ret += StringPrintf("grad norm1 %13.9f\n", grad_.asum_data());
     for(Param* p: GetParams())
       ret += StringPrintf("param id %2d, name %10s,\
           value norm1 %13.9f, grad norm1 %13.9f\n",
@@ -68,41 +65,41 @@ void ConcateLayer::Setup(const LayerProto& proto, int npartitions) {
   grad_.Reshape(shape);
 }
 
-void ConcateLayer::ComputeFeature(Phase phase, Metric *perf){
+void ConcateLayer::ComputeFeature(int flag, Metric *perf){
   LOG(FATAL) << "Not implemented for Concate Layer";
 }
 
-void ConcateLayer::ComputeGradient(Phase phase){
+void ConcateLayer::ComputeGradient(int flag){
   LOG(FATAL) << "Not implemented for Concate Layer";
 }
 
 /************* Implementation for ParserLayer ***********/
-void ParserLayer::ComputeFeature(Phase phase, Metric *perf){
+void ParserLayer::ComputeFeature(int flag, Metric *perf){
   CHECK_EQ(srclayers_.size(),1);
   auto datalayer=static_cast<DataLayer*>(*srclayers_.begin());
-  ParseRecords(phase, datalayer->records(), &data_);
+  ParseRecords(flag, datalayer->records(), &data_);
 }
 
 /************* Implementation for PrefetchLayer ***********/
-void PrefetchLayer::Prefetch(Phase phase){
+void PrefetchLayer::Prefetch(int flag){
   //clock_t s=clock();
   for(auto layer: sublayers_)
-    layer->ComputeFeature(phase, nullptr);
+    layer->ComputeFeature(flag, nullptr);
   //LOG(ERROR)<<(clock()-s)*1.0/CLOCKS_PER_SEC;
 }
 
-void PrefetchLayer::ComputeFeature(Phase phase, Metric* perf){
+void PrefetchLayer::ComputeFeature(int flag, Metric* perf){
   if(thread_.joinable())
     thread_.join();
   else{
-    Prefetch(phase);
+    Prefetch(flag);
   }
   for(auto layer: sublayers_){
     if(layer->is_parserlayer())
       // TODO replace CopyFrom with Swap?
       datablobs_.at(layer->name()).CopyFrom(layer->data(this));
   }
-  thread_=std::thread(&PrefetchLayer::Prefetch, this, phase);
+  thread_=std::thread(&PrefetchLayer::Prefetch, this, flag);
 }
 
 void PrefetchLayer::Setup(const LayerProto& proto, int npartitions) {
@@ -133,7 +130,7 @@ void PrefetchLayer::Setup(const LayerProto& proto, int npartitions) {
       datablobs_[layer->name()]=Blob<float>(layer->data(this).shape());
 }
 
-const Blob<float>& PrefetchLayer::data(const Layer* from, Phase phase) const {
+const Blob<float>& PrefetchLayer::data(const Layer* from) const {
   LOG(FATAL) << " needs update";
   if(from != nullptr) {
     return datablobs_.at("");
@@ -143,7 +140,7 @@ const Blob<float>& PrefetchLayer::data(const Layer* from, Phase phase) const {
   }
 }
 
-Blob<float>* PrefetchLayer::mutable_data(const Layer* from, Phase phase) {
+Blob<float>* PrefetchLayer::mutable_data(const Layer* from) {
   LOG(FATAL) << " needs update";
   if(from!=nullptr){
     return &(datablobs_.at(""));
@@ -194,7 +191,7 @@ int SliceLayer::SliceID(const Layer* layer) const {
   return -1;
 }
 
-const Blob<float>& SliceLayer::data(const Layer* layer, Phase phase) const {
+const Blob<float>& SliceLayer::data(const Layer* layer) const {
   if(layer==nullptr)
     return data_;
   return datavec_[SliceID(layer)];
@@ -204,7 +201,7 @@ const Blob<float>& SliceLayer::grad(const Layer* layer) const {
     return grad_;
   return gradvec_[SliceID(layer)];
 }
-Blob<float>* SliceLayer::mutable_data(const Layer* layer, Phase phase) {
+Blob<float>* SliceLayer::mutable_data(const Layer* layer) {
   if(layer==nullptr)
     return &data_;
   return &datavec_[SliceID(layer)];
@@ -214,7 +211,7 @@ Blob<float>* SliceLayer::mutable_grad(const Layer* layer){
     return &grad_;
   return &gradvec_[SliceID(layer)];
 }
-void SliceLayer::ComputeFeature(Phase phase, Metric *perf) {
+void SliceLayer::ComputeFeature(int flag, Metric *perf) {
   CHECK_EQ(srclayers_.size(),1);
   if(slice_dim_==0){
     const auto& blob=srclayers_.at(0)->data(this);
@@ -226,7 +223,7 @@ void SliceLayer::ComputeFeature(Phase phase, Metric *perf) {
     }
   }
 }
-void SliceLayer::ComputeGradient(Phase phase) {
+void SliceLayer::ComputeGradient(int flag) {
   // LOG(FATAL) << "Not implemented";
 }
 
@@ -240,11 +237,11 @@ void SplitLayer::Setup(const LayerProto& proto, int npartitions) {
   grad_.Reshape(srclayers_[0]->data(this).shape());
 }
 
-void SplitLayer::ComputeFeature(Phase phase, Metric *perf) {
+void SplitLayer::ComputeFeature(int flag, Metric *perf) {
   LOG(FATAL) << "Not implemented";
 
 }
-void SplitLayer::ComputeGradient(Phase phase) {
+void SplitLayer::ComputeGradient(int flag) {
   LOG(FATAL) << "Not implemented";
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/src/neuralnet/layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc
index b5c986e..29a2312 100644
--- a/src/neuralnet/layer.cc
+++ b/src/neuralnet/layer.cc
@@ -72,11 +72,11 @@ void ConvolutionLayer::Setup(const LayerProto& proto, int npartitions) {
 
   weight_ = Param::Create(proto.param(0));
   bias_ = Param::Create(proto.param(1));
-  weight_->Setup(proto.param(0), vector<int>{num_filters_, col_height_});
-  bias_->Setup(proto.param(1), vector<int>{num_filters_});
+  weight_->Setup(vector<int>{num_filters_, col_height_});
+  bias_->Setup(vector<int>{num_filters_});
 }
 
-void ConvolutionLayer::ComputeFeature(Phase phase, Metric* perf){
+void ConvolutionLayer::ComputeFeature(int flag, Metric* perf){
   auto src = Tensor4(srclayers_[0]->mutable_data(this));
   auto data = Tensor3(&data_);
   auto col = Tensor2(&col_data_);
@@ -93,7 +93,7 @@ void ConvolutionLayer::ComputeFeature(Phase phase, Metric* perf){
   data+=broadcast<1>(bias, data.shape);
 }
 
-void ConvolutionLayer::ComputeGradient(Phase phase) {
+void ConvolutionLayer::ComputeGradient(int flag) {
   auto src = Tensor4(srclayers_[0]->mutable_data(this));
   auto col = Tensor2(&col_data_);
   auto weight = Tensor2(weight_->mutable_data());
@@ -137,9 +137,9 @@ void DropoutLayer::Setup(const LayerProto& proto, int npartitions) {
   pdrop_ = proto.dropout_conf().dropout_ratio();
 }
 
-void DropoutLayer::ComputeFeature(Phase phase, Metric* perf) {
+void DropoutLayer::ComputeFeature(int flag, Metric* perf) {
   // check training
-  if(phase != kTrain){//!training){
+  if((flag & kTrain) != kTrain) {
     data_.CopyFrom(srclayers_[0]->data(this));
     return;
   }
@@ -152,7 +152,7 @@ void DropoutLayer::ComputeFeature(Phase phase, Metric* perf) {
   data = src * mask;
 }
 
-void DropoutLayer::ComputeGradient(Phase phase)  {
+void DropoutLayer::ComputeGradient(int flag)  {
   auto mask = Tensor1(&mask_);
   auto grad = Tensor1(&grad_);
   auto gsrc = Tensor1(srclayers_[0]->mutable_grad(this));
@@ -164,94 +164,69 @@ RBMVisLayer::~RBMVisLayer() {
   delete bias_;
 }
 
-void RBMVisLayer::Setup(const LayerProto& proto,
-      int npartitions) {
+void RBMVisLayer::Setup(const LayerProto& proto, int npartitions) {
   Layer::Setup(proto, npartitions);
   CHECK_EQ(srclayers_.size(), 2);
-  // hid_idx_: index indicating which srclayer is is hidden layer
-  // data_idx_: index indicating which srclayer is data layer
-  for (unsigned int i = 0; i < srclayers_.size(); i++)
-    for (unsigned int j = 0; j < (srclayers_[i]-> dstlayers()).size(); j++)
-      if (strcmp(((srclayers_[i]->dstlayers()).at(j)->name().c_str()),
-        (this->name()).c_str()) == 0)
-        hid_idx_ = i;
-  for (unsigned int i = 0; i < srclayers_.size(); i++)
-    if (i != static_cast<unsigned int>(hid_idx_) )
-      data_idx_ = i;
-  const auto& src = srclayers_[data_idx_]->data(this);
-  is_first_iteration_vis_ = true;
+  hid_layer_ = nullptr;
+  for (auto src : srclayers_) {
+    for (auto dst : src->srclayers()) {
+      if (dst->name() == name()) {
+        CHECK(hid_layer_ == nullptr);
+        hid_layer_ = static_cast<RBMHidLayer*>(src);
+      }
+    }
+  }
+  input_layer_ = srclayers_[0] != hid_layer_ ? srclayers_[0]: srclayers_[1];
+  const auto& src = input_layer_->data(this);
   batchsize_ = src.shape()[0];
-  neg_batchsize_ = batchsize_;
-  /*gibbs sampling size and input have the same size*/
-  vdim_ = src.count()/batchsize_;
-  hdim_ = proto.rbmvis_conf().num_output();
-  data_.Reshape(vector<int>{batchsize_, vdim_});  // this is visible dimension
-  vis_sample_.Reshape(vector<int>{neg_batchsize_, vdim_});
+  data_.ReshapeLike(src);  // this is visible dimension
+  neg_data_.ReshapeLike(data_);
+  neg_sample_.ReshapeLike(data_);
   weight_ = Param::Create(proto.param(0));
   bias_ = Param::Create(proto.param(1));
-  weight_->Setup(proto.param(0), vector<int>{hdim_, vdim_});
-  bias_->Setup(proto.param(1), vector<int>{vdim_});
-}
-
-void RBMVisLayer::ComputeFeature(Phase phase, Metric* perf) {
-  if (phase == kPositive) { /*positive phase*/
-    auto data = Tensor2(&data_);
-    CHECK_EQ(srclayers_[data_idx_]->data(this).count(), batchsize_*vdim_);
-    auto src = Tensor2(srclayers_[data_idx_]->mutable_data(this));
-    Copy(data, src);
-  } else if (phase == kNegative) {   /*negative phase*/
-      auto hid_sample =
-        Tensor2(srclayers_[hid_idx_]->mutable_data(this, kNegative));
-      // fetch sampling results from hidden layer
-      auto vis_sample = Tensor2(&vis_sample_);
-      auto weight = Tensor2(weight_->mutable_data());
-      auto bias = Tensor1(bias_->mutable_data());
-      vis_sample = dot(hid_sample, weight);
-      vis_sample+=repmat(bias, neg_batchsize_);
-      vis_sample = F<op::sigmoid>(vis_sample);
+  bias_->Setup(vector<int>{src.count() / batchsize_});
+}
+Blob<float>* RBMVisLayer::Sample(int flag) {
+  Tensor<cpu, 2> sample, data;
+  if ((flag & kPositive) == kPositive) {
+    LOG(FATAL) << "RBMVisLayer can not be sampled for positive flag";
+  } else {
+    data = Tensor2(&neg_data_);
+    sample = Tensor2(&neg_sample_);
+  }
+  auto random = TSingleton<Random<cpu>>::Instance();
+  random->SampleBinary(sample, data);
+  return &neg_sample_;
+}
+void RBMVisLayer::ComputeFeature(int flag, Metric* perf) {
+  if ((flag & kPositive) == kPositive) { /*positive flag*/
+    data_.CopyFrom(input_layer_->data(this), true);
+  } else if ((flag & kNegative) == kNegative) {   /*negative flag*/
+    auto hid_sample = Tensor2(hid_layer_->Sample(flag));
+    // fetch sampling results from hidden layer
+    auto data = Tensor2(&neg_data_);
+    auto weight = Tensor2(weight_->mutable_data());
+    auto bias = Tensor1(bias_->mutable_data());
+    data = dot(hid_sample, weight);
+    data += repmat(bias, batchsize_);
+    data = F<op::sigmoid>(data);
+    if ((flag & kTest) == kTest) {
+      const float *dptr = data_.cpu_data(), *rcns = neg_data_.cpu_data();
+      float err = 0.f;
+      for (int i = 0; i < data_.count(); i++) {
+        err += (dptr[i] - rcns[i]) * (dptr[i] - rcns[i]);
+      }
+      perf->Add("Squared Error", err / batchsize_);
     }
+  }
 }
 
-void RBMVisLayer::ComputeGradient(Phase phase) {
-  auto data = Tensor2(&data_);
-  auto hid_data = Tensor2(srclayers_[hid_idx_]->mutable_data(this, kPositive));
-  auto vis_sample = Tensor2(&vis_sample_);
-  auto hid_sample =
-       Tensor2(srclayers_[hid_idx_]->mutable_data(this, kNegative));
-  // fetch sampling results from hidden layer
-  auto gweight = Tensor2(weight_->mutable_grad());
-  auto gbias = Tensor1(bias_->mutable_grad());
-  gbias = sum_rows(vis_sample);
-  gbias -= sum_rows(data);
-  gweight = dot(hid_sample.T(), vis_sample);
-  gweight -= dot(hid_data.T(), data);
-  gbias*=(1.0f)/(1.0f*batchsize_);
-  gweight*=(1.0f)/(1.0f*batchsize_);
-}
-
-void RBMVisLayer::ComputeLoss(Metric* perf) {
-  float loss_sqr = (0.0f);
-  CHECK_EQ(srclayers_[data_idx_]->data(this).count(), batchsize_*vdim_);
-  auto src = Tensor2(srclayers_[data_idx_]->mutable_data(this));
-  auto hid_data = Tensor2(srclayers_[hid_idx_]->mutable_data(this, kPositive));
-  // gibbs using u
-  auto weight = Tensor2(weight_->mutable_data());
-  auto bias = Tensor1(bias_->mutable_data());
-  Tensor<cpu, 2> reconstruct(Shape2(batchsize_, vdim_)); /*reconstruct error*/
-  AllocSpace(reconstruct);
-  reconstruct = dot(hid_data, weight);
-  reconstruct+=repmat(bias, batchsize_);
-  reconstruct = F<op::sigmoid>(reconstruct);
-  float *src_dptr = src.dptr;
-  for (int i = 0; i < vdim_*batchsize_; i++) {
-      int recon_row = i / vdim_;
-      int recon_col = i - recon_row * vdim_;
-      loss_sqr += (src_dptr[i] - reconstruct[recon_row][recon_col]) *
-                  (src_dptr[i] - reconstruct[recon_row][recon_col]);
-  }
-  FreeSpace(reconstruct);
-  perf->Reset();
-  perf->Add("sqr_reconstruct_error", loss_sqr);
+void RBMVisLayer::ComputeGradient(int flag) {
+  auto vis_pos = Tensor2(&data_);
+  auto vis_neg = Tensor2(&neg_data_);
+    auto gbias = Tensor1(bias_->mutable_grad());
+  gbias = sum_rows(vis_neg);
+  gbias -= sum_rows(vis_pos);
 }
 /**************** Implementation for RBMHidLayer********************/
 RBMHidLayer::~RBMHidLayer() {
@@ -263,84 +238,75 @@ void RBMHidLayer::Setup(const LayerProto& proto,
       int npartitions) {
   Layer::Setup(proto, npartitions);
   CHECK_EQ(srclayers_.size(), 1);
-  const auto& src_data = srclayers_[0]->data(this, kPositive);
-  const auto& src_sample = srclayers_[0]->data(this, kNegative);
-  scale_ = static_cast<float> (1.0f);
+  const auto& src_data = srclayers_[0]->data(this);
   batchsize_ = src_data.shape()[0];
-  neg_batchsize_ = src_sample.shape()[0];
   vdim_ = src_data.count()/batchsize_;
   hdim_ = proto.rbmhid_conf().hid_dim();
   gaussian_ = proto.rbmhid_conf().gaussian();
   data_.Reshape(vector<int>{batchsize_, hdim_});
-  hid_sample_.Reshape(vector<int>{neg_batchsize_, hdim_});
+  neg_data_.ReshapeLike(data_);
+  sample_.ReshapeLike(data_);
+  neg_sample_.ReshapeLike(data_);
   weight_ = Param::Create(proto.param(0));
   bias_ = Param::Create(proto.param(1));
-  bias_->Setup(proto.param(1), vector<int>{hdim_});
-  weight_->Setup(proto.param(0), vector<int>{hdim_, vdim_});
+  bias_->Setup(vector<int>{hdim_});
+  weight_->Setup(vector<int>{hdim_, vdim_});
+  vis_layer_ = static_cast<RBMVisLayer*> (srclayers_[0]);
 }
 
-void RBMHidLayer::ComputeFeature(Phase phase, Metric* perf) {
-  if (phase == kPositive) {  /*postive phase*/
-    auto data = Tensor2(&data_);
-
-    auto hid_sample = Tensor2(&hid_sample_);
-
-    CHECK_EQ(srclayers_[0]->data(this, kPositive).count(), batchsize_*vdim_);
-    auto src = Tensor2(srclayers_[0]->mutable_data(this, kPositive));
-    auto weight = Tensor2(weight_->mutable_data());
-    auto bias = Tensor1(bias_->mutable_data());
-    data = dot(src, weight.T());
-    data += repmat(bias, batchsize_);
-
-    if (!gaussian_)
-      data = F<op::sigmoid>(data);
+Blob<float>* RBMHidLayer::Sample(int flag) {
+  Tensor<cpu, 2> sample, data;
+  if ((flag & kPositive) == kPositive) {
+    data = Tensor2(&data_);
+    sample = Tensor2(&sample_);
+  } else {
+    data = Tensor2(&neg_data_);
+    sample = Tensor2(&neg_sample_);
+  }
+  auto random = TSingleton<Random<cpu>>::Instance();
+  if (gaussian_) {  // first gibbs
+    random->SampleGaussian(sample, 0.0f, 1.0f);
+    sample += data;
+  } else {
+    random->SampleBinary(sample, data);
+  }
+  return (flag & kPositive) == kPositive ? &sample_ : &neg_sample_;
+}
 
-    Copy(hid_sample, data);
+void RBMHidLayer::ComputeFeature(int flag, Metric* perf) {
+  auto weight = Tensor2(weight_->mutable_data());
+  auto bias = Tensor1(bias_->mutable_data());
 
-    if (gaussian_) {  // first gibbs
-      Tensor<cpu, 2> gaussian_sample(Shape2(batchsize_, hdim_));
-      AllocSpace(gaussian_sample);
-      auto random = TSingleton<Random<cpu>>::Instance();
-      random->SampleGaussian(gaussian_sample, 0.0f, 1.0f);
-      hid_sample += gaussian_sample;
-      FreeSpace(gaussian_sample);
-    } else {
-        TSingleton<Random<cpu>>::Instance()->SampleBinary(hid_sample);
-    }
+  Tensor<cpu, 2> data, src;
+  if ((flag & kPositive) == kPositive) {  /*postive flag*/
+    data = Tensor2(&data_);
+    src = Tensor2(vis_layer_->mutable_data(this));
+  } else {
+    data = Tensor2(&neg_data_);
+    src = Tensor2(vis_layer_->Sample(flag));
+  }
+  data = dot(src, weight.T());
+  data += repmat(bias, batchsize_);
 
-  } else if (phase == kNegative) {   /*negative phase*/
-      CHECK_EQ(srclayers_[0]->data(this, kNegative).count(),
-         neg_batchsize_*vdim_);
-      auto src_sample = Tensor2(srclayers_[0]->mutable_data(this, kNegative));
-      auto hid_sample = Tensor2(&hid_sample_);
-      auto bias = Tensor1(bias_->mutable_data());
-      auto weight = Tensor2(weight_->mutable_data());
-      hid_sample = dot(src_sample, weight.T());
-      hid_sample += repmat(bias, neg_batchsize_);
-      if (!gaussian_)
-        hid_sample = F<op::sigmoid>(hid_sample);
-    } else if (phase == kLoss) {   /*test phase*/
-        auto data = Tensor2(&data_);  // data: sigmoid(Wv+b)
-        if (gaussian_) {
-          Tensor<cpu, 2> gaussian_sample(Shape2(batchsize_, hdim_));
-          AllocSpace(gaussian_sample);
-          auto random = TSingleton<Random<cpu>>::Instance();
-          random->SampleGaussian(gaussian_sample, 0.0f, 1.0f);
-          data += gaussian_sample;
-          FreeSpace(gaussian_sample);
-        }
-        else
-          TSingleton<Random<cpu>>::Instance()->SampleBinary(data);
-      }
+  if (!gaussian_)
+    data = F<op::sigmoid>(data);
 }
 
-void RBMHidLayer::ComputeGradient(Phase phase) {
-  auto data = Tensor2(&data_);
-  auto hid_sample = Tensor2(&hid_sample_);
+void RBMHidLayer::ComputeGradient(int flag) {
+  auto hid_pos = Tensor2(&data_);
+  auto hid_neg = Tensor2(&neg_data_);
+  auto vis_pos = Tensor2(vis_layer_->mutable_data(this));
+  auto vis_neg = Tensor2(vis_layer_->mutable_data(this));
+
   auto gbias = Tensor1(bias_->mutable_grad());
-  gbias = sum_rows(hid_sample);
-  gbias -= sum_rows(data);
-  gbias *= scale_/(1.0f*batchsize_);
+  gbias = sum_rows(hid_neg);
+  gbias -= sum_rows(hid_pos);
+  gbias /= batchsize_;
+
+  auto gweight = Tensor2(weight_->mutable_grad());
+  gweight = dot(hid_neg.T(), vis_neg);
+  gweight -= dot(hid_pos.T(), vis_pos);
+  gweight /= batchsize_;
 }
 /*********** Implementation for InnerProductLayer**********/
 InnerProductLayer::~InnerProductLayer() {
@@ -362,13 +328,13 @@ void InnerProductLayer::Setup(const LayerProto& proto, int npartitions) {
   weight_ = Param::Create(proto.param(0));
   bias_ = Param::Create(proto.param(1));
   if (transpose_)
-    weight_->Setup(proto.param(0), vector<int>{vdim_, hdim_});
+    weight_->Setup(vector<int>{vdim_, hdim_});
   else
-    weight_->Setup(proto.param(0), vector<int>{hdim_, vdim_});
-  bias_->Setup(proto.param(1), vector<int>{hdim_});
+    weight_->Setup(vector<int>{hdim_, vdim_});
+  bias_->Setup(vector<int>{hdim_});
 }
 
-void InnerProductLayer::ComputeFeature(Phase phase, Metric* perf) {
+void InnerProductLayer::ComputeFeature(int flag, Metric* perf) {
   auto data = Tensor2(&data_);
   auto src = Tensor2(srclayers_[0]->mutable_data(this));
   auto weight = Tensor2(weight_->mutable_data());
@@ -381,7 +347,7 @@ void InnerProductLayer::ComputeFeature(Phase phase, Metric* perf) {
   data+=repmat(bias, batchsize_);
 }
 
-void InnerProductLayer::ComputeGradient(Phase phas) {
+void InnerProductLayer::ComputeGradient(int phas) {
   auto src = Tensor2(srclayers_[0]->mutable_data(this));
   auto grad = Tensor2(&grad_);
   auto weight = Tensor2(weight_->mutable_data());
@@ -411,7 +377,7 @@ void LabelLayer::Setup(const LayerProto& proto, int npartitions){
   data_.Reshape(vector<int>{batchsize});
 }
 
-void LabelLayer::ParseRecords(Phase phase, const vector<Record>& records,
+void LabelLayer::ParseRecords(int flag, const vector<Record>& records,
     Blob<float>* blob){
   int rid=0;
   float *label= blob->mutable_cpu_data() ;
@@ -442,7 +408,7 @@ void LRNLayer::Setup(const LayerProto& proto, int npartitions) {
   width_=s[3];
 }
 
-void LRNLayer::ComputeFeature(Phase phase, Metric* perf) {
+void LRNLayer::ComputeFeature(int flag, Metric* perf) {
   const float salpha = alpha_ / lsize_;
   auto src = Tensor4(srclayers_[0]->mutable_data(this));
   auto data = Tensor4(&data_);
@@ -452,7 +418,7 @@ void LRNLayer::ComputeFeature(Phase phase, Metric* perf) {
   data = src * F<op::power>(norm, -beta_ );
 }
 
-void LRNLayer::ComputeGradient(Phase phase) {
+void LRNLayer::ComputeGradient(int flag) {
   const float salpha = alpha_ / lsize_;
   auto src = Tensor4(srclayers_[0]->mutable_data(this));
   auto norm = Tensor4(&norm_);
@@ -466,8 +432,10 @@ void LRNLayer::ComputeGradient(Phase phase) {
 
 /**************** Implementation for MnistImageLayer******************/
 
-void MnistLayer::ParseRecords(Phase phase,
+void MnistLayer::ParseRecords(int flag,
     const vector<Record>& records, Blob<float>* blob){
+  if ((flag & kForward) == 0)
+    return;
   LOG_IF(ERROR, records.size()==0)<<"Empty records to parse";
   int ndim=records.at(0).image().shape_size();
   int inputsize =records.at(0).image().shape(ndim-1);
@@ -554,7 +522,7 @@ void PoolingLayer::Setup(const LayerProto& proto, int npartitions) {
   grad_.ReshapeLike(data_);
 }
 
-void PoolingLayer::ComputeFeature(Phase phase, Metric* perf) {
+void PoolingLayer::ComputeFeature(int flag, Metric* perf) {
   auto src = Tensor4(srclayers_[0]->mutable_data(this));
   auto data = Tensor4(&data_);
   if(pool_ == PoolingProto_PoolMethod_MAX)
@@ -567,7 +535,7 @@ void PoolingLayer::ComputeFeature(Phase phase, Metric* perf) {
  * partition only on num/channel dim
  * assume grad and data have the same paritition
  */
-void PoolingLayer::ComputeGradient(Phase phase) {
+void PoolingLayer::ComputeGradient(int flag) {
   auto src = Tensor4(srclayers_[0]->mutable_data(this));
   auto gsrc = Tensor4(srclayers_[0]->mutable_grad(this));
   auto data = Tensor4(&data_);
@@ -587,13 +555,13 @@ void ReLULayer::Setup(const LayerProto& proto, int npartitions) {
   grad_.ReshapeLike(*(srclayers_[0]->mutable_grad(this)));
 }
 
-void ReLULayer::ComputeFeature(Phase phase, Metric* perf) {
+void ReLULayer::ComputeFeature(int flag, Metric* perf) {
   auto data = Tensor1(&data_);
   auto src = Tensor1(srclayers_[0]->mutable_data(this));
   data=F<op::relu>(src);
 }
 
-void ReLULayer::ComputeGradient(Phase phase) {
+void ReLULayer::ComputeGradient(int flag) {
   auto data = Tensor1(&data_);
   auto grad = Tensor1(&grad_);
   auto gsrc = Tensor1(srclayers_[0]->mutable_grad(this));
@@ -602,8 +570,11 @@ void ReLULayer::ComputeGradient(Phase phase) {
 
 /*************** Implementation for RGBImageLayer *************************/
 
-void RGBImageLayer::ParseRecords(Phase phase,
+void RGBImageLayer::ParseRecords(int flag,
     const vector<Record>& records, Blob<float>* blob){
+  if ((flag & kForward) == 0)
+    return;
+
   const vector<int>& s=blob->shape();
   auto images = Tensor4(&data_);
   const SingleLabelImageRecord& r=records.at(0).image();
@@ -617,8 +588,8 @@ void RGBImageLayer::ParseRecords(Phase phase,
   const float* meandptr=mean_.cpu_data();
   for(const Record& record: records){
     auto image=images[rid];
-    bool do_crop=cropsize_>0&&(phase == kTrain);
-    bool do_mirror=mirror_&&rand()%2&&(phase == kTrain);
+    bool do_crop = cropsize_ > 0 && ((flag & kTrain) == kTrain);
+    bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain);
     float* dptr=nullptr;
     if(do_crop||do_mirror)
       dptr=raw_image.dptr;
@@ -697,7 +668,10 @@ void RGBImageLayer::Setup(const LayerProto& proto, int npartitions) {
 }
 
 /***************Implementation for ShardDataLayer**************************/
-void ShardDataLayer::ComputeFeature(Phase phase, Metric* perf){
+void ShardDataLayer::ComputeFeature(int flag, Metric* perf){
+  if ((flag & kForward) == 0)
+    return;
+
   if (shard_ == nullptr)
     shard_ = new DataShard(layer_proto_.sharddata_conf().path(),
         DataShard::kRead);
@@ -747,13 +721,13 @@ void SigmoidLayer::Setup(const LayerProto& proto, int npartitions) {
   grad_.ReshapeLike(srclayers_[0]->grad(this));
 }
 
-void SigmoidLayer::ComputeFeature(Phase phase, Metric* perf) {
+void SigmoidLayer::ComputeFeature(int flag, Metric* perf) {
   auto data = Tensor1(&data_);
   auto src = Tensor1(srclayers_[0]->mutable_data(this));
   data = F<op::sigmoid>(src);
 }
 
-void SigmoidLayer::ComputeGradient(Phase phase) {
+void SigmoidLayer::ComputeGradient(int flag) {
   auto data = Tensor1(&data_);
   auto grad = Tensor1(&grad_);
   auto gsrc = Tensor1(srclayers_[0]->mutable_grad(this));
@@ -766,13 +740,13 @@ void TanhLayer::Setup(const LayerProto& proto, int npartitions){
   grad_.ReshapeLike(srclayers_[0]->grad(this));
 }
 
-void TanhLayer::ComputeFeature(Phase phase, Metric* perf) {
+void TanhLayer::ComputeFeature(int flag, Metric* perf) {
   auto data = Tensor1(&data_);
   auto src = Tensor1(srclayers_[0]->mutable_data(this));
   data=F<op::stanh>(src);
 }
 
-void TanhLayer::ComputeGradient(Phase phase) {
+void TanhLayer::ComputeGradient(int flag) {
   auto data = Tensor1(&data_);
   auto grad = Tensor1(&grad_);
   auto gsrc = Tensor1(srclayers_[0]->mutable_grad(this));
@@ -787,7 +761,7 @@ void EuclideanLossLayer::Setup(const LayerProto& proto, int npartitions) {
   dim_ = data_.count()/batchsize_;
   metric_.Reshape(vector<int>{1});
 }
-void EuclideanLossLayer::ComputeFeature(Phase phase, Metric* perf) {
+void EuclideanLossLayer::ComputeFeature(int flag, Metric* perf) {
   const float* reconstruct_dptr = srclayers_[0]->data(this).cpu_data();
   const float* input_dptr = srclayers_[1]->data(this).cpu_data();
   float loss = 0;
@@ -805,7 +779,7 @@ void EuclideanLossLayer::ComputeFeature(Phase phase, Metric* perf) {
       srclayers_[1]->data(this).cpu_data() + (batchsize_*dim_));
   perf->Add("loss", loss/(1.0f*batchsize_));
 }
-void EuclideanLossLayer::ComputeGradient(Phase phase) {
+void EuclideanLossLayer::ComputeGradient(int flag) {
   const float* reconstruct_dptr = srclayers_[0]->data(this).cpu_data();
   const float* input_dptr = srclayers_[1]->data(this).cpu_data();
   Blob<float>* gsrcblob = srclayers_[0]->mutable_grad(this);
@@ -828,7 +802,7 @@ void SoftmaxLossLayer::Setup(const LayerProto& proto, int npartitions) {
   metric_.Reshape(vector<int>{2});
   scale_=proto.softmaxloss_conf().scale();
 }
-void SoftmaxLossLayer::ComputeFeature(Phase phase, Metric* perf) {
+void SoftmaxLossLayer::ComputeFeature(int flag, Metric* perf) {
   Shape<2> s=Shape2(batchsize_, dim_);
   Tensor<cpu, 2> prob(data_.mutable_cpu_data(), s);
   Tensor<cpu, 2> src(srclayers_[0]->mutable_data(this)->mutable_cpu_data(), s);
@@ -863,7 +837,7 @@ void SoftmaxLossLayer::ComputeFeature(Phase phase, Metric* perf) {
   perf->Add("accuracy", precision*scale_/(1.0f*batchsize_));
 }
 
-void SoftmaxLossLayer::ComputeGradient(Phase phase) {
+void SoftmaxLossLayer::ComputeGradient(int flag) {
   const float* label=srclayers_[1]->data(this).cpu_data();
   Blob<float>* gsrcblob=srclayers_[0]->mutable_grad(this);
   gsrcblob->CopyFrom(data_);

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 7f030ac..1c79aea 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -21,18 +21,12 @@ message JobProto {
   required string name = 1;
   // neural net consits of a set of connected layers
   required NetProto neuralnet = 3;
-  // algorithms calculating gradients for one mini-batch/iteration
-  optional TrainOneBatchAlg alg = 5 [default = kUserAlg];
-  // user defined algorithm
-  optional string user_alg = 6;
+  // algorithm for computing gradients over one mini-batch
+  required AlgProto train_one_batch = 5;
   // configuration of SGD updater, including learning rate, etc.
   required UpdaterProto updater = 7;
   // cluster toplogy conf
   required ClusterProto cluster = 9;
-
-  // for setting CD fields
-  optional CDProto cd_conf = 12;
-
   // total num of steps for training
   required int32 train_steps = 16;
   // frequency of displaying training info
@@ -86,6 +80,16 @@ message JobProto {
 // Protos used by JobProto
 // -----------------------
 
+message AlgProto {
+  // algorithms calculating gradients for one mini-batch/iteration
+  optional AlgType alg = 1 [default = kUserAlg];
+  // user defined algorithm
+  optional string user_alg = 2;
+  // for setting CD fields
+  optional CDProto cd_conf = 10;
+
+  extensions 101 to 200;
+}
 message NetProto {
   repeated LayerProto layer = 1;
   // partitioning type for parallelism
@@ -140,7 +144,7 @@ message ClusterProto {
 
 message CDProto {
   //number of steps for gibbs sampling
-  optional int32 pcd_k = 1 [default = 1];
+  optional int32 cd_k = 1 [default = 1];
 }
 
 message LayerProto {
@@ -182,8 +186,6 @@ message LayerProto {
   optional PrefetchProto prefetch_conf = 44;
   // configuration for rbmhid layer
   optional RBMHidProto rbmhid_conf = 49;
-  // configuration for rbmvis layer
-  optional RBMVisProto rbmvis_conf = 48;
   // configuration for rectified linear unit layer
   optional ReLUProto relu_conf = 38;
   // configuration for rgb image parser layer
@@ -365,11 +367,6 @@ message DropoutProto {
   optional float dropout_ratio = 30 [default = 0.5];
 }
 
-message RBMVisProto {
-  optional int32 num_output = 1; // The number of outputs for the layer
-  optional bool bias_term = 2 [default = true]; // whether to have bias terms
-}
-
 message RBMHidProto {
   optional int32 hid_dim = 1; // The number of outputs for the layer
   optional bool bias_term = 2 [default = true]; // whether to have bias terms
@@ -559,16 +556,16 @@ enum PartitionType {
 }
 
 enum Phase {
-  kTrain = 0;
-  kValidation = 1;
-  kTest= 2;
+  kTrain = 1;
+  kValidation = 2;
+  kTest= 4;
   // postivie phase for contrastive divergence algorithm
-  kPositive = 3;
+  kPositive = 8;
   // negative phase for contrastive divergence algorithm
-  kNegative = 4;
-  kForward = 5;
-  kBackward = 6;
-  kLoss = 7;
+  kNegative = 16;
+  kForward = 32;
+  kBackward = 64;
+  kLoss = 128;
 }
 
 enum ParamType {
@@ -578,7 +575,7 @@ enum ParamType {
   kUser = 103;
 }
 
-enum TrainOneBatchAlg {
+enum AlgType {
   // Back-propagation algorithm for feed-forward models, e.g., CNN and RNN
   kBP = 1;
   // Contrastive Divergence algorithm for RBM, DBM, etc.

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/src/trainer/worker.cc
----------------------------------------------------------------------
diff --git a/src/trainer/worker.cc b/src/trainer/worker.cc
index f112b17..a22a8ef 100644
--- a/src/trainer/worker.cc
+++ b/src/trainer/worker.cc
@@ -13,12 +13,14 @@ using std::thread;
 Worker* Worker::Create(const JobProto& proto) {
   auto factory = Singleton<Factory<singa::Worker>>::Instance();
   Worker* worker = nullptr;
-  if (proto.has_user_alg())
-    worker = factory->Create(proto.user_alg());
+  const auto& conf = proto.train_one_batch();
+  if (conf.has_user_alg())
+    worker = factory->Create(conf.user_alg());
   else
-    worker = factory->Create(proto.alg());
+    worker = factory->Create(conf.alg());
   return worker;
 }
+
 void Worker::Init(int thread_id, int grp_id, int id) {
   thread_id_ = thread_id;
   grp_id_ = grp_id;
@@ -63,7 +65,7 @@ void Worker::InitLocalParams() {
     // the param from previous checkpoint files will be overwritten by
     // the param with the same name in later checkpoint files.
     for (const auto checkpoint : job_conf_.checkpoint_path()) {
-      LOG(INFO) << "Load from checkpoint file " << checkpoint;
+      LOG(ERROR) << "Load from checkpoint file " << checkpoint;
       BlobProtos bps;
       ReadProtoFromBinaryFile(checkpoint.c_str(), &bps);
       for (int i = 0; i < bps.name_size(); i++) {
@@ -342,11 +344,11 @@ void BPWorker::Forward(
           Collect(p, step);
         }
       }
-      layer->ComputeFeature(phase, perf);
+      layer->ComputeFeature(phase | kForward, perf);
       if (layer->is_bridgesrclayer())  // send data to other workers
         SendBlobs(true, false, static_cast<BridgeLayer*>(layer), net);
       if (DisplayDebugInfo(step))
-        LOG(INFO) << layer->DebugString(step, kForward);
+        LOG(INFO) << layer->DebugString(step, phase | kForward);
     }
   }
 }
@@ -359,9 +361,9 @@ void BPWorker::Backward(int step, shared_ptr<NeuralNet> net) {
       if(layer->is_bridgesrclayer()) {
         // ReceiveBlobs(false, true, layer, net);
       }
-      layer->ComputeGradient(kTrain);
+      layer->ComputeGradient(kTrain | kBackward);
       if (DisplayDebugInfo(step))
-        LOG(INFO) << layer->DebugString(step, kBackward);
+        LOG(INFO) << layer->DebugString(step, kTrain | kBackward);
       for (Param* p : layer->GetParams())
         Update(p, step);
       if (layer->is_bridgedstlayer()) {
@@ -381,72 +383,34 @@ void BPWorker::TestOneBatch(int step, Phase phase,
   Forward(step, phase, net, perf);
 }
 /****************************CDWorker**********************************/
-void CDWorker::Init(int thread_id, int group_id, int worker_id) {
-  Worker::Init(thread_id, group_id, worker_id);
-}
-
-void CDWorker::PositivePhase(int step,
-     shared_ptr<NeuralNet> net, Metric* perf) {
-  auto& layers = net->layers();
-  // LOG(ERROR)<<"Positive Phase";
-  for (auto& layer : layers) {
-    for (Param* p : layer->GetParams()) {  // wait until param is updated
+void CDWorker::TrainOneBatch(int step, Metric* perf) {
+  const auto& layers = train_net_->layers();
+  for (auto* layer : layers) {
+    for (Param* p : layer->GetParams())  // wait until param is updated
       Collect(p, step);
-    }
-    layer->ComputeFeature(kPositive, perf);
+    layer->ComputeFeature(kPositive | kForward, perf);
   }
-}
-
-void CDWorker::NegativePhase(int step,
-     shared_ptr<NeuralNet> net, Metric* perf) {
-// for negative phase, gibbs sampling only concerns RBM bottom and top layer
-  auto& layers = net->layers();
-  // LOG(ERROR)<<"Negative Phase";
-    for (auto& layer : layers) {
-      if (layer->is_vislayer() || layer->is_hidlayer()) {
-        layer->ComputeFeature(kNegative, perf);
-      }
-    }
-}
-
-void CDWorker::GradientPhase(int step, shared_ptr<NeuralNet> net) {
-  auto& layers = net->layers();
-  // LOG(ERROR)<<"Gradient Phase";
-  for (auto& layer : layers) {
-    if (layer->is_vislayer() || layer->is_hidlayer()) {
-      layer->ComputeGradient(kTrain);
-      for (Param* p : layer->GetParams()) {
-        Update(p, step);
-      }
+  for (auto* layer : layers)
+    layer->ComputeFeature(kNegative | kTest, perf);
+  for (int i = 1; i < job_conf_.train_one_batch().cd_conf().cd_k(); i++) {
+    for (auto* layer : layers) {
+      layer->ComputeFeature(kNegative, perf);
     }
   }
-}
-
-void CDWorker::LossPhase(int step, shared_ptr<NeuralNet> net, Metric* perf) {
-  auto& layers = net->layers();
-  // LOG(ERROR)<<"Loss Phase";
-  for (auto& layer : layers) {
-    if (layer->is_hidlayer()) {
-      layer->ComputeFeature(kLoss, perf);
+  for (auto* layer : layers) {
+    layer->ComputeGradient(kTrain);
+    for (Param* p : layer->GetParams()) {
+      Update(p, step);
     }
   }
-  for (auto& layer : layers) {
-    if (layer->is_vislayer()) {
-      layer->ComputeLoss(perf);
-    }
-  }
-}
-
-void CDWorker::TrainOneBatch(int step, Metric* perf) {
-  PositivePhase(step, train_net_, perf);
-  NegativePhase(step, train_net_, perf);
-  GradientPhase(step, train_net_);
-  LossPhase(step, train_net_, perf);
 }
 
 void CDWorker::TestOneBatch(int step, Phase phase,
-     shared_ptr<NeuralNet> net, Metric* perf) {
-  PositivePhase(step, test_net_, perf);
-  LossPhase(step, test_net_, perf);
+    shared_ptr<NeuralNet> net, Metric* perf) {
+  auto& layers = net->layers();
+  for (auto layer : layers)
+    layer->ComputeFeature(kPositive | kForward, perf);
+  for (auto layer : layers)
+    layer->ComputeFeature(kNegative | kTest, perf);
 }
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/src/utils/common.cc
----------------------------------------------------------------------
diff --git a/src/utils/common.cc b/src/utils/common.cc
index 1888380..d13faea 100644
--- a/src/utils/common.cc
+++ b/src/utils/common.cc
@@ -40,7 +40,7 @@ string IntVecToString(const vector<int>& vec) {
  *  * Formatted string.
  *   */
 string VStringPrintf(string fmt, va_list l) {
-  char buffer[32768];
+  char buffer[4096];
   vsnprintf(buffer, sizeof(buffer), fmt.c_str(), l);
   return string(buffer);
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fbbcaafd/src/utils/param.cc
----------------------------------------------------------------------
diff --git a/src/utils/param.cc b/src/utils/param.cc
index 67f14ab..69f697b 100644
--- a/src/utils/param.cc
+++ b/src/utils/param.cc
@@ -73,17 +73,18 @@ Param* Param::Create(const ParamProto& proto) {
     p = factory->Create(proto.user_type());
   else
     p = factory->Create(proto.type());
+  p->Init(proto);
   return p;
 }
 
 Param::Param():local_version_(-1), slice_start_(0), num_slices_(0),
   num_pending_requests_(0), data_(nullptr) {
 }
-void Param::Setup(const ParamProto& proto, const vector<int>& shape) {
+
+void Param::Setup(const vector<int>& shape) {
   data_ = std::make_shared<Blob<float>>(shape);
   grad_.Reshape(shape);
   history_.Reshape(shape);
-  proto_.CopyFrom(proto);
 }
 
 void Param::AddSlice(int slice_id, int size) {
@@ -178,7 +179,8 @@ Msg* Param::HandlePutMsg(Msg** msg, bool reserve) {
   proto.set_lr_scale(lr);
   proto.set_wd_scale(wc);
   vector<int> shape{size};
-  Setup(proto, shape);
+  Init(proto);
+  Setup(shape);
   if (ptr == nullptr) {
     CHECK((*msg)->NextFrame());
     CHECK_EQ(size* sizeof(float), (*msg)->FrameSize());
@@ -298,6 +300,8 @@ void Param::ShareFrom(const Param& other) {
           other.data_->shape().begin()));
   }
   data_ = other.data_;
+  if (grad_.count() == 0)
+    grad_.Reshape(data_->shape());
   slice_offset_ = other.slice_offset_;
   slice_size_ = other.slice_size_;
   slice_start_ = other.slice_start_;


Mime
View raw message