singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [2/4] incubator-singa git commit: SINGA-111 Add slice, concate and split layers
Date Thu, 10 Dec 2015 13:12:17 GMT
SINGA-111 Add slice, concate and split layers

Add concate layer implementation and test case


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3ea1eb69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3ea1eb69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3ea1eb69

Branch: refs/heads/master
Commit: 3ea1eb691708b39b4a786f6494f56f6dbfe8e1df
Parents: 28e48a6
Author: WANG Sheng <wangsheng1001@gmail.com>
Authored: Tue Dec 8 21:28:30 2015 +0800
Committer: WANG Sheng <wangsheng1001@gmail.com>
Committed: Thu Dec 10 16:22:59 2015 +0800

----------------------------------------------------------------------
 .../singa/neuralnet/connection_layer/concate.h  |   3 +
 src/neuralnet/connection_layer/concate.cc       |  52 ++++++--
 src/test/test_connection_layers.cc              | 119 +++++++++++++++++++
 3 files changed, 164 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/include/singa/neuralnet/connection_layer/concate.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/concate.h b/include/singa/neuralnet/connection_layer/concate.h
index 5875835..6e40040 100644
--- a/include/singa/neuralnet/connection_layer/concate.h
+++ b/include/singa/neuralnet/connection_layer/concate.h
@@ -37,6 +37,9 @@ class ConcateLayer : public ConnectionLayer {
   void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
   void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
   void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+  int concate_dim_;
 };
 
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/src/neuralnet/connection_layer/concate.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc
index fd2fb24..8f519d2 100644
--- a/src/neuralnet/connection_layer/concate.cc
+++ b/src/neuralnet/connection_layer/concate.cc
@@ -26,30 +26,62 @@ namespace singa {
 using std::vector;
 
 void ConcateLayer::Setup(const LayerProto& conf,
-    const vector<Layer*>& srclayers) {
-  Layer::Setup(conf, srclayers);
-  size_t concate_dim = conf.concate_conf().concate_dim();
-  CHECK_GE(concate_dim, 0);
+                         const vector<Layer*>& srclayers) {
   CHECK_GT(srclayers.size(), 1);
+  Layer::Setup(conf, srclayers);
+  concate_dim_ = conf.concate_conf().concate_dim();
   vector<int> shape = srclayers[0]->data(this).shape();
+  CHECK_GE(concate_dim_, 0);
+  CHECK_LT(concate_dim_, shape.size());
   for (size_t i = 1; i < srclayers.size(); i++) {
-    const vector<int>& srcshape = srclayers[i]->data(this).shape();
+    const vector<int>& src_shape = srclayers[i]->data(this).shape();
     for (size_t j = 0; j < shape.size(); j++)
-      if (j == concate_dim)
-        shape[j] += srcshape[j];
+      if (static_cast<int>(j) == concate_dim_)
+        shape[j] += src_shape[j];
       else
-        CHECK_EQ(shape[j], srcshape[j]);
+        CHECK_EQ(shape[j], src_shape[j]);
   }
   data_.Reshape(shape);
   grad_.Reshape(shape);
 }
 
 void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
-  LOG(FATAL) << "Not implemented for Concate Layer";
+  CHECK_GT(srclayers.size(), 1);
+  // calculate step for each memcpy
+  int step = srclayers[0]->data(this).shape()[concate_dim_];
+  for (unsigned i = concate_dim_ + 1; i < data_.shape().size(); ++i)
+    step *= data_.shape()[i];
+  int srclayer_offset = 0;
+  int concate_offset = 0;
+  while (concate_offset < data_.count()) {
+    for (size_t i = 0; i < srclayers.size(); ++i) {
+      const float* src = srclayers[i]->data(this).cpu_data() + srclayer_offset;
+      float* dst = data_.mutable_cpu_data() + concate_offset;
+      memcpy(dst, src, step * sizeof(float));
+      concate_offset += step;
+    }
+    srclayer_offset += step;
+  }
 }
 
 void ConcateLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
-  LOG(FATAL) << "Not implemented for Concate Layer";
+  CHECK_GT(srclayers.size(), 1);
+  // calculate step for each memcpy
+  int step = srclayers[0]->grad(this).shape()[concate_dim_];
+  for (unsigned i = concate_dim_ + 1; i < grad_.shape().size(); ++i)
+    step *= grad_.shape()[i];
+  int srclayer_offset = 0;
+  int concate_offset = 0;
+  while (concate_offset < grad_.count()) {
+    for (size_t i = 0; i < srclayers.size(); ++i) {
+      const float* src = grad_.cpu_data() + concate_offset;
+      float* dst = srclayers[i]->mutable_grad(this)->mutable_cpu_data()
+                   + srclayer_offset;
+      memcpy(dst, src, step * sizeof(float));
+      concate_offset += step;
+    }
+    srclayer_offset += step;
+  }
 }
 
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index a2cb1d3..e072114 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -26,6 +26,7 @@
 #include "singa/comm/msg.h"
 #include "singa/comm/socket.h"
 #include "singa/neuralnet/connection_layer/bridge.h"
+#include "singa/neuralnet/connection_layer/concate.h"
 #include "singa/neuralnet/connection_layer/slice.h"
 #include "singa/neuralnet/neuron_layer/dummy.h"
 #include "singa/proto/job.pb.h"
@@ -277,3 +278,121 @@ TEST(ConnectionLayerTest, ModelSliceTest) {
               out[(i / step) % K].grad(nullptr).cpu_data()[offset + i % step]);
   }
 }
+
+TEST(ConnectionLayerTest, DataConcateTest) {
+  // use dummy as input layers
+  LayerProto proto_in[K];
+  vector<Layer*> src_in[K];
+  DummyLayer in[K];
+  for (int i = 0; i < K; ++i) {
+    proto_in[i].set_name("dummy_input_"+std::to_string(i));
+    proto_in[i].set_partition_id(i);
+    proto_in[i].mutable_dummy_conf()->set_input(true);
+    proto_in[i].mutable_dummy_conf()->add_shape(N / K);
+    proto_in[i].mutable_dummy_conf()->add_shape(M);
+    in[i].Setup(proto_in[i], src_in[i]);
+  }
+
+  // add concate layer
+  vector<Layer*> src_concate;
+  for (int i = 0; i < K; ++i)
+    src_concate.push_back(static_cast<Layer*>(&in[i]));
+  LayerProto proto_concate;
+  proto_concate.set_name("concate");
+  proto_concate.mutable_concate_conf()->set_concate_dim(0);
+  ConcateLayer concate;
+  concate.Setup(proto_concate, src_concate);
+  ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
+  ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(1), M);
+
+  // use dummy as output layer
+  vector<Layer*> src_out;
+  src_out.push_back(static_cast<Layer*>(&concate));
+  LayerProto proto_out;
+  proto_out.set_name("dummy_output");
+  proto_out.mutable_dummy_conf()->set_output(true);
+  DummyLayer out;
+  out.Setup(proto_out, src_out);
+
+  // test for computing feature
+  for (int i = 0; i < K; ++i)
+    in[i].ComputeFeature(0, src_in[i]);
+  concate.ComputeFeature(0, src_concate);
+  out.ComputeFeature(0, src_out);
+  int step = (N * M) / K;
+  for (int i = 0; i < out.data(nullptr).count(); ++i) {
+    ASSERT_EQ(in[i / step].data(nullptr).cpu_data()[i % step],
+              out.data(nullptr).cpu_data()[i]);
+  }
+
+  // test for computing gradient
+  out.ComputeGradient(0, src_out);
+  concate.ComputeGradient(0, src_concate);
+  for (int i = 0; i < K; ++i)
+    in[i].ComputeGradient(0, src_in[i]);
+  for (int i = 0; i < out.grad(nullptr).count(); ++i) {
+    ASSERT_EQ(in[i / step].grad(nullptr).cpu_data()[i % step],
+              out.grad(nullptr).cpu_data()[i]);
+  }
+}
+
+TEST(ConnectionLayerTest, ModelConcateTest) {
+  // use dummy as input layers
+  LayerProto proto_in[K];
+  vector<Layer*> src_in[K];
+  DummyLayer in[K];
+  for (int i = 0; i < K; ++i) {
+    proto_in[i].set_name("dummy_input_"+std::to_string(i));
+    proto_in[i].set_partition_id(i);
+    proto_in[i].mutable_dummy_conf()->set_input(true);
+    proto_in[i].mutable_dummy_conf()->add_shape(N);
+    proto_in[i].mutable_dummy_conf()->add_shape(M / K);
+    in[i].Setup(proto_in[i], src_in[i]);
+  }
+
+  // add concate layer
+  vector<Layer*> src_concate;
+  for (int i = 0; i < K; ++i)
+    src_concate.push_back(static_cast<Layer*>(&in[i]));
+  LayerProto proto_concate;
+  proto_concate.set_name("concate");
+  proto_concate.mutable_concate_conf()->set_concate_dim(1);
+  ConcateLayer concate;
+  concate.Setup(proto_concate, src_concate);
+  ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
+  ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(1), M);
+
+  // use dummy as output layer
+  vector<Layer*> src_out;
+  src_out.push_back(static_cast<Layer*>(&concate));
+  LayerProto proto_out;
+  proto_out.set_name("dummy_output");
+  proto_out.mutable_dummy_conf()->set_output(true);
+  DummyLayer out;
+  out.Setup(proto_out, src_out);
+
+  // test for computing feature
+  for (int i = 0; i < K; ++i)
+    in[i].ComputeFeature(0, src_in[i]);
+  concate.ComputeFeature(0, src_concate);
+  out.ComputeFeature(0, src_out);
+  int step = M / K;
+  int offset = 0;
+  for (int i = 0; i < out.grad(nullptr).count(); ++i) {
+    if (i && i % M == 0) offset += step;
+    ASSERT_EQ(in[(i / step) % K].data(nullptr).cpu_data()[offset + i % step],
+              out.data(nullptr).cpu_data()[i]);
+  }
+
+  // test for computing gradient
+  out.ComputeGradient(0, src_out);
+  concate.ComputeGradient(0, src_concate);
+  for (int i = 0; i < K; ++i)
+    in[i].ComputeGradient(0, src_in[i]);
+  offset = 0;
+  for (int i = 0; i < out.grad(nullptr).count(); ++i) {
+    if (i && i % M == 0) offset += step;
+    ASSERT_EQ(in[(i / step) % K].grad(nullptr).cpu_data()[offset + i % step],
+              out.grad(nullptr).cpu_data()[i]);
+  }
+}


Mime
View raw message