singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [19/21] incubator-singa git commit: SINGA-65 Add an example of implementing user's own layers
Date Wed, 16 Sep 2015 04:19:58 GMT
SINGA-65 Add an example of implementing user's own layers

The folder examples/mlp includes files for a user-defined layer, i.e.,
HiddenLayer class. The main.cc register this class. The myproto.proto defines
the configuration of this layer and extends it to the base LayerProto.
Makefile.example contains instructions for compiling and linking
user code.

job.conf specifies a simple model with only one hidden layer.
deep.conf configures a deep model with 6 hidden layers, which is slower
per iteration, but converges faster than the simple model.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/4820d5e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/4820d5e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/4820d5e7

Branch: refs/heads/tutorial
Commit: 4820d5e757395915e0f4064ff6a0df304b4fea45
Parents: 78dcab9
Author: wangwei <wangwei@comp.nus.edu.sg>
Authored: Thu Sep 3 09:57:03 2015 +0800
Committer: Wei Wang <wangwei@comp.nus.edu.sg>
Committed: Wed Sep 16 11:39:16 2015 +0800

----------------------------------------------------------------------
 examples/cifar10/job.conf     |  25 +++--
 examples/mlp/Makefile.example |   8 ++
 examples/mlp/deep.conf        | 221 +++++++++++++++++++++++++++++++++++++
 examples/mlp/hidden_layer.cc  |  72 ++++++++++++
 examples/mlp/hidden_layer.h   |  22 ++++
 examples/mlp/job.conf         |  95 ++++++++++++++++
 examples/mlp/main.cc          |  25 +++++
 examples/mlp/myproto.proto    |  11 ++
 src/utils/tool.cc             |   1 +
 9 files changed, 468 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index 343d969..3f09c3f 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -2,25 +2,26 @@ name: "cifar10-convnet"
 train_steps: 1000
 test_steps: 100
 test_freq:300
-disp_freq: 30
-train_one_batch {
+disp_freq:30
+train_one_batch
+{
   alg: kBP
 }
-updater{
+updater   {
   type: kSGD
   weight_decay:0.004
   learning_rate {
-    type: kFixedStep
-    fixedstep_conf:{
-      step:0
-      step:60000
-      step:65000
-      step_lr:0.001
-      step_lr:0.0001
-      step_lr:0.00001
-    }
+  type: kFixedStep
+  fixedstep_conf:{
+    step:0
+    step:60000
+    step:65000
+    step_lr:0.001
+    step_lr:0.0001
+    step_lr:0.00001
   }
 }
+}
 neuralnet {
   layer{
     name: "data"

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/mlp/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mlp/Makefile.example b/examples/mlp/Makefile.example
new file mode 100644
index 0000000..71edd71
--- /dev/null
+++ b/examples/mlp/Makefile.example
@@ -0,0 +1,8 @@
+# refer to examples/mnist/ for data creation
+MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
+
+all:
+	protoc --proto_path=../../src/proto --proto_path=. --cpp_out=. myproto.proto
+	$(CXX) main.cc hidden_layer.cc myproto.pb.cc $(MSHADOW_FLAGS) -std=c++11 -lsinga -lglog
-lprotobuf -lopenblas -I../../include\
+		-I../../include/proto/ -L../../.libs/ -L/usr/local  -Wl,-unresolved-symbols=ignore-in-shared-libs
-Wl,-rpath=../../.libs/\
+		-o mlp.bin

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/mlp/deep.conf
----------------------------------------------------------------------
diff --git a/examples/mlp/deep.conf b/examples/mlp/deep.conf
new file mode 100644
index 0000000..588fefa
--- /dev/null
+++ b/examples/mlp/deep.conf
@@ -0,0 +1,221 @@
+name: "mlp"
+train_steps: 1200
+test_steps:10
+test_freq:60
+disp_freq:10
+train_one_batch {
+  alg: kBP
+}
+updater{
+  type: kSGD
+  learning_rate{
+    type : kStep
+    base_lr: 0.001
+    step_conf{
+      change_freq: 60
+      gamma: 0.997
+    }
+  }
+}
+neuralnet {
+  layer {
+    name: "data"
+    type: kShardData
+    sharddata_conf {
+      path: "examples/mnist/mnist_train_shard"
+      batchsize: 1000
+    }
+    exclude: kTest
+  }
+
+  layer {
+    name: "data"
+    type: kShardData
+    sharddata_conf {
+      path: "examples/mnist/mnist_test_shard"
+      batchsize: 1000
+    }
+    exclude: kTrain
+  }
+
+  layer{
+    name:"mnist"
+    type: kMnist
+    srclayers: "data"
+    mnist_conf {
+      norm_a: 127.5
+      norm_b: 1
+    }
+  }
+  layer{
+    name: "label"
+    type: kLabel
+    srclayers: "data"
+  }
+
+  layer{
+    name: "hid1"
+    user_type: "kHidden"
+    srclayers:"mnist"
+    [singa.hidden_conf] {
+      num_output: 2500
+    }
+    param{
+      name: "w1"
+      init {
+        type: kUniform
+        low:-0.05
+        high:0.05
+      }
+    }
+    param{
+      name: "b1"
+      init {
+        type : kUniform
+        low: -0.05
+        high:0.05
+      }
+    }
+  }
+
+  layer{
+    name: "hid2"
+    user_type: "kHidden"
+    srclayers:"hid1"
+    [singa.hidden_conf] {
+      num_output: 2000
+    }
+    param{
+      name: "w2"
+      init {
+        type: kUniform
+        low:-0.05
+        high:0.05
+      }
+    }
+    param{
+      name: "b2"
+      init {
+        type : kUniform
+        low: -0.05
+        high:0.05
+      }
+    }
+  }
+
+  layer{
+    name: "hid3"
+    user_type: "kHidden"
+    srclayers:"hid2"
+    [singa.hidden_conf] {
+      num_output: 1500
+    }
+    param{
+      name: "w3"
+      init {
+        type: kUniform
+        low:-0.05
+        high:0.05
+      }
+    }
+    param{
+      name: "b3"
+      init {
+        type : kUniform
+        low: -0.05
+        high:0.05
+      }
+    }
+  }
+
+  layer{
+    name: "hid4"
+    user_type: "kHidden"
+    srclayers:"hid3"
+    [singa.hidden_conf] {
+      num_output: 1000
+    }
+    param{
+      name: "w4"
+      init {
+        type: kUniform
+        low:-0.05
+        high:0.05
+      }
+    }
+    param{
+      name: "b4"
+      init {
+        type : kUniform
+        low: -0.05
+        high:0.05
+      }
+    }
+  }
+
+
+  layer{
+    name: "hid5"
+    user_type: "kHidden"
+    srclayers:"hid4"
+    [singa.hidden_conf] {
+      num_output: 500
+    }
+    param{
+      name: "w5"
+      init {
+        type: kUniform
+        low:-0.05
+        high:0.05
+      }
+    }
+    param{
+      name: "b5"
+      init {
+        type : kUniform
+        low: -0.05
+        high:0.05
+      }
+    }
+  }
+
+
+  layer{
+    name: "hid6"
+    user_type: "kHidden"
+    srclayers:"hid5"
+    [singa.hidden_conf] {
+      num_output: 10
+    }
+    param{
+      name: "w6"
+      init {
+        type: kUniform
+        low:-0.05
+        high:0.05
+      }
+    }
+    param{
+      name: "b6"
+      init {
+        type : kUniform
+        low: -0.05
+        high:0.05
+      }
+    }
+  }
+ layer{
+    name: "loss"
+    type:kSoftmaxLoss
+    softmaxloss_conf{
+      topk:1
+    }
+    srclayers:"hid6"
+    srclayers:"label"
+  }
+}
+cluster {
+  nworker_groups: 1
+  nserver_groups: 1
+  workspace: "examples/mnist"
+}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/mlp/hidden_layer.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/hidden_layer.cc b/examples/mlp/hidden_layer.cc
new file mode 100644
index 0000000..f4dacf2
--- /dev/null
+++ b/examples/mlp/hidden_layer.cc
@@ -0,0 +1,72 @@
+#include "hidden_layer.h"
+
+#include "mshadow/tensor.h"
+#include "mshadow/cxxnet_op.h"
+
+namespace singa {
+using namespace mshadow;
+using mshadow::cpu;
+using mshadow::Shape1;
+using mshadow::Shape2;
+
+inline Tensor<cpu, 2> NewTensor2(Blob<float>* blob) {
+  const vector<int>& shape = blob->shape();
+  Tensor<cpu, 2> tensor(blob->mutable_cpu_data(),
+      Shape2(shape[0], blob->count() / shape[0]));
+  return tensor;
+}
+
+inline Tensor<cpu, 1> NewTensor1(Blob<float>* blob) {
+  Tensor<cpu, 1> tensor(blob->mutable_cpu_data(), Shape1(blob->count()));
+  return tensor;
+}
+
+
+HiddenLayer::~HiddenLayer() {
+  delete weight_;
+  delete bias_;
+}
+
+void HiddenLayer::Setup(const LayerProto& proto, int npartitions) {
+  Layer::Setup(proto, npartitions);
+  CHECK_EQ(srclayers_.size(), 1);
+  const auto& src = srclayers_[0]->data(this);
+  batchsize_ = src.shape()[0];
+  vdim_ = src.count() / batchsize_;
+  hdim_ = layer_proto_.GetExtension(hidden_conf).num_output();
+  data_.Reshape(vector<int>{batchsize_, hdim_});
+  grad_.ReshapeLike(data_);
+  weight_ = Param::Create(proto.param(0));
+  bias_ = Param::Create(proto.param(1));
+  weight_->Setup(vector<int>{hdim_, vdim_});
+  bias_->Setup(vector<int>{hdim_});
+}
+
+void HiddenLayer::ComputeFeature(int flag, Metric* perf) {
+  auto data = NewTensor2(&data_);
+  auto src = NewTensor2(srclayers_[0]->mutable_data(this));
+  auto weight = NewTensor2(weight_->mutable_data());
+  auto bias = NewTensor1(bias_->mutable_data());
+  data = dot(src, weight.T());
+  // repmat: repeat bias vector into batchsize rows
+  data += expr::repmat(bias, batchsize_);
+  data = expr::F<op::stanh>(data);
+}
+
+void HiddenLayer::ComputeGradient(int flag, Metric* perf) {
+  auto data = NewTensor2(&data_);
+  auto src = NewTensor2(srclayers_[0]->mutable_data(this));
+  auto grad = NewTensor2(&grad_);
+  auto weight = NewTensor2(weight_->mutable_data());
+  auto gweight = NewTensor2(weight_->mutable_grad());
+  auto gbias = NewTensor1(bias_->mutable_grad());
+
+  grad = expr::F<op::stanh_grad>(data) * grad;
+  gbias = expr::sum_rows(grad);
+  gweight = dot(grad.T(), src);
+  if (srclayers_[0]->mutable_grad(this) != nullptr) {
+    auto gsrc = NewTensor2(srclayers_[0]->mutable_grad(this));
+    gsrc = dot(grad, weight);
+  }
+}
+}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/mlp/hidden_layer.h
----------------------------------------------------------------------
diff --git a/examples/mlp/hidden_layer.h b/examples/mlp/hidden_layer.h
new file mode 100644
index 0000000..c305261
--- /dev/null
+++ b/examples/mlp/hidden_layer.h
@@ -0,0 +1,22 @@
+#include "neuralnet/layer.h"
+#include "myproto.pb.h"
+
+namespace singa {
+class HiddenLayer : public NeuronLayer {
+ public:
+  ~HiddenLayer();
+  void Setup(const LayerProto& proto, int npartitions) override;
+  void ComputeFeature(int flag, Metric* perf) override;
+  void ComputeGradient(int flag, Metric* perf) override;
+  const std::vector<Param*> GetParams() const override {
+    std::vector<Param*> params{weight_, bias_};
+    return params;
+  }
+
+ private:
+  int batchsize_;
+  int vdim_, hdim_;
+  bool transpose_;
+  Param *weight_, *bias_;
+};
+}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/mlp/job.conf
----------------------------------------------------------------------
diff --git a/examples/mlp/job.conf b/examples/mlp/job.conf
new file mode 100644
index 0000000..163bc19
--- /dev/null
+++ b/examples/mlp/job.conf
@@ -0,0 +1,95 @@
+name: "mlp"
+train_steps: 1200
+test_steps:10
+test_freq:60
+disp_freq:10
+train_one_batch {
+  alg: kBP
+}
+updater{
+  type: kSGD
+  learning_rate{
+    type : kStep
+    base_lr: 0.001
+    step_conf{
+      change_freq: 60
+      gamma: 0.997
+    }
+  }
+}
+neuralnet {
+  layer {
+    name: "data"
+    type: kShardData
+    sharddata_conf {
+      path: "examples/mnist/mnist_train_shard"
+      batchsize: 1000
+    }
+    exclude: kTest
+  }
+
+  layer {
+    name: "data"
+    type: kShardData
+    sharddata_conf {
+      path: "examples/mnist/mnist_test_shard"
+      batchsize: 1000
+    }
+    exclude: kTrain
+  }
+
+  layer{
+    name:"mnist"
+    type: kMnist
+    srclayers: "data"
+    mnist_conf {
+      norm_a: 127.5
+      norm_b: 1
+    }
+  }
+  layer{
+    name: "label"
+    type: kLabel
+    srclayers: "data"
+  }
+
+
+  layer{
+    name: "hid1"
+    user_type: "kHidden"
+    srclayers:"mnist"
+    [singa.hidden_conf] {
+      num_output: 10
+    }
+    param{
+      name: "w1"
+      init {
+        type: kUniform
+        low:-0.05
+        high:0.05
+      }
+    }
+    param{
+      name: "b1"
+      init {
+        type : kUniform
+        low: -0.05
+        high:0.05
+      }
+    }
+  }
+ layer{
+    name: "loss"
+    type:kSoftmaxLoss
+    softmaxloss_conf{
+      topk:1
+    }
+    srclayers:"hid1"
+    srclayers:"label"
+  }
+}
+cluster {
+  nworker_groups: 1
+  nserver_groups: 1
+  workspace: "examples/mnist"
+}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/mlp/main.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/main.cc b/examples/mlp/main.cc
new file mode 100644
index 0000000..c27f38d
--- /dev/null
+++ b/examples/mlp/main.cc
@@ -0,0 +1,25 @@
+#include <string>
+#include "singa.h"
+#include "hidden_layer.h"
+#include "myproto.pb.h"
+#include "utils/common.h"
+
+int main(int argc, char **argv) {
+  //  must create driver at the beginning and call its Init method.
+  singa::Driver driver;
+  driver.Init(argc, argv);
+
+  //  if -resume in argument list, set resume to true; otherwise false
+  int resume_pos = singa::ArgPos(argc, argv, "-resume");
+  bool resume = (resume_pos != -1);
+
+  //  users can register new subclasses of layer, updater, etc.
+  driver.RegisterLayer<singa::HiddenLayer, std::string>("kHidden");
+
+  //  get the job conf, and custmize it if need
+  singa::JobProto jobConf = driver.job_conf();
+
+  //  submit the job
+  driver.Submit(resume, jobConf);
+  return 0;
+}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/examples/mlp/myproto.proto
----------------------------------------------------------------------
diff --git a/examples/mlp/myproto.proto b/examples/mlp/myproto.proto
new file mode 100644
index 0000000..ba62ac2
--- /dev/null
+++ b/examples/mlp/myproto.proto
@@ -0,0 +1,11 @@
+package singa;
+import "job.proto";
+
+message HiddenProto {
+  required int32 num_output = 1;
+  optional bool use_stanh = 2 [default = true];
+}
+
+extend LayerProto {
+  optional HiddenProto hidden_conf = 102;
+}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4820d5e7/src/utils/tool.cc
----------------------------------------------------------------------
diff --git a/src/utils/tool.cc b/src/utils/tool.cc
index 305cbe7..d2bf631 100644
--- a/src/utils/tool.cc
+++ b/src/utils/tool.cc
@@ -24,6 +24,7 @@
 #include <algorithm>
 #include <fstream>
 #include <iostream>
+#include <google/protobuf/text_format.h>
 #include "proto/job.pb.h"
 #include "proto/singa.pb.h"
 #include "utils/cluster_rt.h"


Mime
View raw message