singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [1/3] incubator-singa git commit: SINGA-190 - Add prelu layer and flatten layer
Date Sun, 12 Jun 2016 14:35:11 GMT
Repository: incubator-singa
Updated Branches:
  refs/heads/dev 6d69047ad -> 26df5ac03


SINGA-190 - Add prelu layer and flatten layer

Implement prelu layer and flatten layer for cpu version.

Write gtest for prelu and flatten layer.

Pass all tests.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5afd81b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5afd81b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5afd81b7

Branch: refs/heads/dev
Commit: 5afd81b7f4841b15ce292b5b2e3e26c25a79b912
Parents: 04e23d1
Author: jixin <jixin@comp.nus.edu.sg>
Authored: Wed Jun 8 15:58:09 2016 +0800
Committer: jixin <jixin@comp.nus.edu.sg>
Committed: Sat Jun 11 16:46:59 2016 +0800

----------------------------------------------------------------------
 src/model/layer/flatten.cc |  62 ++++++++
 src/model/layer/flatten.h  |  54 +++++++
 src/model/layer/prelu.cc   | 169 +++++++++++++++++++++
 src/model/layer/prelu.h    |  60 ++++++++
 src/proto/model.proto      | 321 ++++++++++++++++++----------------------
 test/singa/test_flatten.cc | 156 +++++++++++++++++++
 test/singa/test_prelu.cc   | 149 +++++++++++++++++++
 7 files changed, 793 insertions(+), 178 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/flatten.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/flatten.cc b/src/model/layer/flatten.cc
new file mode 100644
index 0000000..3ed37fe
--- /dev/null
+++ b/src/model/layer/flatten.cc
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/model/layer.h"
+#include "./flatten.h"
+namespace singa {
+
+void Flatten::Setup(const LayerConf &conf) {
+  Layer::Setup(conf);
+  axis_ = conf.flatten_conf().axis();
+}
+
+const Tensor Flatten::Forward(int flag, const Tensor &input) {
+  Tensor output = input;
+  input_shape_ = input.shape();
+  if (!Axis()) {
+    // reshape to 1D
+    size_t dim = output.Size();
+    output.Reshape(Shape {
+      dim
+    });
+    output_shape_ = Shape { dim }
+    ;
+  } else {
+    // reshape to 2D
+    size_t dim1 = 1, dim2;
+    for (int i = 0; i < Axis(); i++)
+      dim1 *= output.shape(i);
+    dim2 = output.Size() / dim1;
+    output.Reshape(Shape {
+      dim1, dim2
+    });
+    output_shape_ = Shape { dim1, dim2 }
+    ;
+  }
+  return output;
+}
+
+const std::pair<Tensor, vector<Tensor> > Flatten::Backward(int flag,
+                                                           const Tensor &grad) {
+  vector<Tensor> param_grad;
+  Tensor input_grad = grad;
+  input_grad.Reshape(Input_shape());
+  return std::make_pair(input_grad, param_grad);
+}
+
+} // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/flatten.h
----------------------------------------------------------------------
diff --git a/src/model/layer/flatten.h b/src/model/layer/flatten.h
new file mode 100644
index 0000000..cb36542
--- /dev/null
+++ b/src/model/layer/flatten.h
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_MODEL_LAYER_FLATTEN_H_
+#define SRC_MODEL_LAYER_FLATTEN_H_
+#include <utility>
+#include <string>
+#include <vector>
+#include "singa/model/layer.h"
+
+namespace singa {
+class Flatten : public Layer {
+public:
+  /// \copydoc Layer::layer_type();
+  const std::string layer_type() const override { return "Flatten"; }
+
+  /// \copydoc Layer::Setup(const LayerConf&);
+  void Setup(const LayerConf &conf) override;
+
+  /// \copydoc Layer::Forward(int flag, const Tensor&);
+  const Tensor Forward(int flag, const Tensor &input) override;
+
+  /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&);
+  const std::pair<Tensor, vector<Tensor> > Backward(int flag,
+                                                    const Tensor &grad)
+      override;
+
+  const int Axis() const { return axis_; }
+  const Shape Input_shape() const { return input_shape_; }
+  const Shape Output_shape() const { return output_shape_; }
+
+protected:
+  /// flatten layer reshape the input to 2D, one from 0 to axis_-1, one from
+  /// axis_ to end.
+  /// if axis_ is 0, reshape the input to 1D.
+  int axis_;
+  Shape input_shape_, output_shape_;
+};
+}      // namespace singa
+#endif // SRC_MODEL_LAYER_FLATTEN_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/prelu.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/prelu.cc b/src/model/layer/prelu.cc
new file mode 100644
index 0000000..1d6a2e7
--- /dev/null
+++ b/src/model/layer/prelu.cc
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/model/layer.h"
+#include "./prelu.h"
+namespace singa {
+
+void PReLU::Setup(const LayerConf &conf) {
+  Layer::Setup(conf);
+  channel_shared_ = conf.prelu_conf().channel_shared();
+  format_ = conf.prelu_conf().format();
+  // Push back params into param_values_
+  for (const auto &spec : conf.param())
+    param_specs_.push_back(spec);
+  param_values_.push_back(&a_);
+}
+
+const Tensor PReLU::Forward(int flag, const Tensor &input) {
+  Tensor output;
+  if (!channel_shared_) {
+    size_t n, c, h, w;
+    Tensor temp = (input <= 0.f);
+    if (temp.nDim() == 4) {
+      if (format_ == "NCHW") {
+        n = temp.shape(0);
+        c = temp.shape(1);
+        h = temp.shape(2);
+        w = temp.shape(3);
+        temp.Reshape(Shape {
+          n *c, h *w
+        });
+        Tensor temp_a(Shape {
+          n, c
+        });
+        Uniform(1.f, 1.f, &temp_a);
+        MultRow(a_, &temp_a);
+        temp_a.Reshape(Shape {
+          n *c
+        });
+        MultColumn(temp_a, &temp);
+      } else if (format_ == "NHWC") {
+        n = temp.shape(0);
+        h = temp.shape(1);
+        w = temp.shape(2);
+        c = temp.shape(3);
+        temp.Reshape(Shape {
+          n *h *w, c
+        });
+        MultRow(a_, &temp);
+      } else {
+        LOG(FATAL) << "Incorrect input format for prelu layer.";
+      }
+    } else {
+      LOG(FATAL) << "Incorrect input format for prelu layer.";
+    }
+    output = input * ((input > 0.f) + temp);
+  } else {
+    // share the first param of Tensor A along all channels
+    const float a = a_.data<const float *>()[0];
+    output = input * ((input > 0.f) + (input <= 0.f) * a);
+  }
+  if (flag & kTrain)
+    buf_.push(input);
+  return output;
+}
+
+const std::pair<Tensor, vector<Tensor> > PReLU::Backward(int flag,
+                                                         const Tensor &grad) {
+  vector<Tensor> param_grad;
+  CHECK(!buf_.empty());
+  Tensor input_grad, input = buf_.top();
+  buf_.pop();
+  Tensor da;
+  da.ResetLike(a_);
+  if (!channel_shared_) {
+    size_t n, c, h, w;
+    Tensor temp1 = (input <= 0.f);
+    if (temp1.nDim() == 4) {
+      if (format_ == "NCHW") {
+        n = temp1.shape(0);
+        c = temp1.shape(1);
+        h = temp1.shape(2);
+        w = temp1.shape(3);
+        temp1.Reshape(Shape {
+          n *c, h *w
+        });
+        Tensor temp_a(Shape {
+          n, c
+        });
+        Uniform(1.f, 1.f, &temp_a);
+        MultRow(a_, &temp_a);
+        temp_a.Reshape(Shape {
+          n *c
+        });
+        MultColumn(temp_a, &temp1);
+        temp1.Reshape(Shape {
+          n, c, h, w
+        });
+      } else if (format_ == "NHWC") {
+        n = temp1.shape(0);
+        h = temp1.shape(1);
+        w = temp1.shape(2);
+        c = temp1.shape(3);
+        temp1.Reshape(Shape {
+          n *h *w, c
+        });
+        MultRow(a_, &temp1);
+        temp1.Reshape(Shape {
+          n, h, w, c
+        });
+      } else {
+        LOG(FATAL) << "Incorrect input format for prelu layer.";
+      }
+    } else {
+      LOG(FATAL) << "Incorrect input format for prelu layer.";
+    }
+    input_grad = grad * input * ((input > 0.f) + temp1);
+    Tensor temp2 = grad * input * (input <= 0.f), temp3(Shape {
+      n *c
+    });
+    if (format_ == "NCHW") {
+      temp2.Reshape(Shape {
+        n *c, h *w
+      });
+      SumColumns(temp2, &temp3);
+      temp3.Reshape(Shape {
+        n, c
+      });
+      SumRows(temp3, &da);
+    } else if (format_ == "NHWC") {
+      temp2.Reshape(Shape {
+        n *h *w, c
+      });
+      SumRows(temp2, &da);
+    }
+  } else {
+    // share the first param of Tensor A along all channels
+    const float a = a_.data<const float *>()[0];
+    input_grad = grad * input * ((input > 0.f) + (input <= 0.f) * a);
+    Tensor temp = grad * input * (input <= 0.f);
+    float sum = Sum<float>(temp);
+    Uniform(1.f, 1.f, &da);
+    da *= sum;
+  }
+  param_grad.push_back(da);
+  return std::make_pair(input_grad, param_grad);
+}
+
+void PReLU::ToDevice(Device *device) {
+  Layer::ToDevice(device);
+  a_.ToDevice(device);
+}
+
+} // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/model/layer/prelu.h
----------------------------------------------------------------------
diff --git a/src/model/layer/prelu.h b/src/model/layer/prelu.h
new file mode 100644
index 0000000..1a01d98
--- /dev/null
+++ b/src/model/layer/prelu.h
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SINGA_MODEL_LAYER_PRELU_H_
+#define SINGA_MODEL_LAYER_PRELU_H_
+#include <utility>
+#include <string>
+#include <vector>
+#include "singa/model/layer.h"
+
+namespace singa {
+class PReLU : public Layer {
+ public:
+  /// \copydoc Layer::layer_type()
+   const std::string layer_type() const override { return "PReLU"; }
+
+  /// \copydoc Layer::Setup(const LayerConf&);
+  void Setup(const LayerConf &conf) override;
+
+  /// \copydoc Layer::Forward(int flag, const Tensor&)
+  const Tensor Forward(int flag, const Tensor &input) override;
+
+  /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&);
+  const std::pair<Tensor, vector<Tensor> > Backward(int flag,
+                                                    const Tensor &grad)
+      override;
+
+  void ToDevice(Device *device);
+
+  const bool Channel_shared() const { return channel_shared_; }
+  const Tensor A() const { return a_; }
+  const std::string Format() const { return format_; }
+
+  void Set_a(Tensor a) {
+    a_.ResetLike(a);
+    a_.CopyData(a);
+  }
+
+ protected:
+  bool channel_shared_;
+  std::string format_; // format_ has two valid value, i.e. NCHW, NHWC
+  Tensor a_; // shape of a_ is 2D, i.e. (channels, 1)
+  std::stack<Tensor> buf_;
+};
+}  // namespace singa
+#endif  // SINGA_MODEL_LAYER_PRELU_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/src/proto/model.proto
----------------------------------------------------------------------
diff --git a/src/proto/model.proto b/src/proto/model.proto
index d368296..1d1f3cf 100644
--- a/src/proto/model.proto
+++ b/src/proto/model.proto
@@ -33,64 +33,59 @@ package singa;
 /// using Python (or C++/Java).
 
 // Specifies the shape (dimensions) of a Blob.
-message BlobShape {
-  repeated int64 dim = 1 [packed = true];
-}
+message BlobShape { repeated int64 dim = 1[packed = true]; }
 
 message BlobProto {
   optional BlobShape shape = 7;
-  repeated float data = 5 [packed = true];
-  repeated float diff = 6 [packed = true];
-  repeated double double_data = 8 [packed = true];
-  repeated double double_diff = 9 [packed = true];
+  repeated float data = 5[packed = true];
+  repeated float diff = 6[packed = true];
+  repeated double double_data = 8[packed = true];
+  repeated double double_diff = 9[packed = true];
 
   // 4D dimensions -- deprecated.  Use "shape" instead.
-  optional int32 num = 1 [default = 0];
-  optional int32 channels = 2 [default = 0];
-  optional int32 height = 3 [default = 0];
-  optional int32 width = 4 [default = 0];
+  optional int32 num = 1[default = 0];
+  optional int32 channels = 2[default = 0];
+  optional int32 height = 3[default = 0];
+  optional int32 width = 4[default = 0];
 }
 
 message FillerConf {
   // The filler type, case insensitive
-  optional string type = 1 [default = 'constant'];
-  optional float value = 2 [default = 0]; // the value in constant filler
-  optional float min = 3 [default = 0]; // the min value in uniform filler
-  optional float max = 4 [default = 1]; // the max value in uniform filler
-  optional float mean = 5 [default = 0]; // the mean value in Gaussian filler
-  optional float std = 6 [default = 1]; // the std value in Gaussian filler
+  optional string type = 1[default = 'constant'];
+  optional float value = 2[default = 0]; // the value in constant filler
+  optional float min = 3[default = 0];   // the min value in uniform filler
+  optional float max = 4[default = 1];   // the max value in uniform filler
+  optional float mean = 5[default = 0];  // the mean value in Gaussian filler
+  optional float std = 6[default = 1];   // the std value in Gaussian filler
   // The expected number of non-zero output weights for a given input in
   // Gaussian filler -- the default -1 means don't perform sparsification.
   /* optional int32 sparse = 7 [default = -1]; */
   // Normalize the filler variance by fan_in, fan_out, or their average.
   // Applies to 'xavier' and 'msra' fillers.
   enum VarianceNorm {
-    FAN_IN = 0;
-    FAN_OUT = 1;
-    AVERAGE = 2;
-  }
-  optional VarianceNorm variance_norm = 8 [default = FAN_IN];
+    FAN_IN = 0; FAN_OUT = 1; AVERAGE = 2;
+  } optional VarianceNorm variance_norm = 8[default = FAN_IN];
 }
 
 /// SINGA message
 message OptimizerConf {
   // case insensitive
-  optional string type = 1 [default = "sgd"];
+  optional string type = 1[default = "sgd"];
 
   // used by RMSprop and Adadelta
-  optional float rho = 2 [default = 0.001];
+  optional float rho = 2[default = 0.001];
 
   // used by Adam and AdamMax
-  optional float beta_1 = 3 [default = 0.9];
-  optional float beta_2 = 4 [default = 0.999];
+  optional float beta_1 = 3[default = 0.9];
+  optional float beta_2 = 4[default = 0.999];
 
   // used by vanilla sgd and nesterov
-  optional float momentum = 5 [default = 0.9];
+  optional float momentum = 5[default = 0.9];
 }
 
 message ConstraintConf {
   // case insensitive to limit the parameter value/gradient scale
-  optional string type = 1 [default = "l2"];
+  optional string type = 1[default = "l2"];
   // e.g., the threshold for limiting the parameter scale.
   optional float threshold = 2;
 }
@@ -98,7 +93,7 @@ message ConstraintConf {
 /// SINGA message
 message RegularizerConf {
   // case insensitive to regularize the parameters, e.g., L2.
-  optional string type = 1 [default = "l2"];
+  optional string type = 1[default = "l2"];
   // e.g., the weight decay for L2 regularizer
   optional float coefficient = 2;
 }
@@ -124,10 +119,10 @@ message ParamSpec {
   */
 
   // The multiplier on the global learning rate for this parameter.
-  optional float lr_mult = 3 [default = 1.0];
+  optional float lr_mult = 3[default = 1.0];
 
   // The multiplier on the global weight decay for this parameter.
-  optional float decay_mult = 4 [default = 1.0];
+  optional float decay_mult = 4[default = 1.0];
 
   // SINGA uses this filed internally. Users just configure the fillers in
   // Layer specific conf message as caffe (style).
@@ -137,14 +132,13 @@ message ParamSpec {
 }
 
 enum Phase {
-  kTrain = 4;
-  kEval = 8;
-}
-// NOTE
-// Update the next available ID when you add a new LayerConf field.
-//
-// LayerConf next available layer-specific ID: 139 (last added: tile_param)
-message LayerConf {
+  kTrain = 4; kEval = 8;
+}
+    // NOTE
+    // Update the next available ID when you add a new LayerConf field.
+    //
+    // LayerConf next available layer-specific ID: 139 (last added: tile_param)
+    message LayerConf {
   optional string name = 1; // the layer name
   optional string type = 2; // the layer type
   /* repeated string bottom = 3; // the name of each bottom blob */
@@ -248,7 +242,8 @@ message TransformationConf {
   optional uint32 crop_size = 3 [default = 0];
   // mean_file and mean_value cannot be specified at the same time
   optional string mean_file = 4;
-  // if specified can be repeated once (would substract it from all the channels)
+  // if specified can be repeated once (would substract it from all the
+channels)
   // or can be repeated the same number of times as channels
   // (would subtract them from the corresponding channel)
   repeated float mean_value = 5;
@@ -265,34 +260,33 @@ message LossConf {
   optional int32 ignore_label = 1;
   // If true, normalize each batch across all instances (including spatial
   // dimesions, but not ignored instances); else, divide by batch size only.
-  optional bool normalize = 2 [default = true];
+  optional bool normalize = 2[default = true];
 }
 
 message MetricConf {
   // When computing accuracy, count as correct by comparing the true label to
   // the top k scoring classes.  By default, only compare to the top scoring
   // class (i.e. argmax).
-  optional uint32 top_k = 1 [default = 1];
+  optional uint32 top_k = 1[default = 1];
 
   // The "label" axis of the prediction blob, whose argmax corresponds to the
   // predicted label -- may be negative to index from the end (e.g., -1 for the
   // last axis).  For example, if axis == 1 and the predictions are
   // (N x C x H x W), the label blob is expected to contain N*H*W ground truth
   // labels with integer values in {0, 1, ..., C-1}.
-  optional int32 axis = 2 [default = 1];
+  optional int32 axis = 2[default = 1];
 
   // If specified, ignore instances with the given label.
   optional int32 ignore_label = 3;
 }
-// Messages that store hyper-parameters used by individual layer types follow, in
+// Messages that store hyper-parameters used by individual layer types follow,
+// in
 // alphabetical order.
 
-
-
 message ArgMaxConf {
   // If true produce pairs (argmax, maxval)
-  optional bool out_max_val = 1 [default = false];
-  optional uint32 top_k = 2 [default = 1];
+  optional bool out_max_val = 1[default = false];
+  optional uint32 top_k = 2[default = 1];
   // The axis along which to maximise -- may be negative to index from the
   // end (e.g., -1 for the last axis).
   // By default ArgMaxLayer maximizes over the flattened trailing dimensions
@@ -305,54 +299,51 @@ message ConcatConf {
   // end (e.g., -1 for the last axis).  Other axes must have the
   // same dimension for all the bottom blobs.
   // By default, ConcatLayer concatenates blobs along the "channels" axis (1).
-  optional int32 axis = 2 [default = 1];
+  optional int32 axis = 2[default = 1];
 
   // DEPRECATED: alias for "axis" -- does not support negative indexing.
-  optional uint32 concat_dim = 1 [default = 1];
+  optional uint32 concat_dim = 1[default = 1];
 }
 
 message ContrastiveLossConf {
   // margin for dissimilar pair
-  optional float margin = 1 [default = 1.0];
+  optional float margin = 1[default = 1.0];
   // The first implementation of this cost did not exactly match the cost of
   // Hadsell et al 2006 -- using (margin - d^2) instead of (margin - d)^2.
   // legacy_version = false (the default) uses (margin - d)^2 as proposed in the
   // Hadsell paper. New models should probably use this version.
   // legacy_version = true uses (margin - d^2). This is kept to support /
   // reproduce existing models and results
-  optional bool legacy_version = 2 [default = false];
+  optional bool legacy_version = 2[default = false];
 }
 
 message ConvolutionConf {
   optional uint32 num_output = 1; // The number of outputs for the layer
-  optional bool bias_term = 2 [default = true]; // whether to have bias terms
+  optional bool bias_term = 2[default = true]; // whether to have bias terms
 
   // Pad, kernel size, and stride are all given as a single value for equal
   // dimensions in all spatial dimensions, or once per spatial dimension.
-  repeated uint32 pad = 3; // The padding size; defaults to 0
+  repeated uint32 pad = 3;         // The padding size; defaults to 0
   repeated uint32 kernel_size = 4; // The kernel size
-  repeated uint32 stride = 6; // The stride; defaults to 1
+  repeated uint32 stride = 6;      // The stride; defaults to 1
 
   // For 2D convolution only, the *_h and *_w versions may also be used to
   // specify both spatial dimensions.
-  optional uint32 pad_h = 9 [default = 0]; // The padding height (2D only)
-  optional uint32 pad_w = 10 [default = 0]; // The padding width (2D only)
-  optional uint32 kernel_h = 11; // The kernel height (2D only)
-  optional uint32 kernel_w = 12; // The kernel width (2D only)
-  optional uint32 stride_h = 13; // The stride height (2D only)
-  optional uint32 stride_w = 14; // The stride width (2D only)
+  optional uint32 pad_h = 9[default = 0];  // The padding height (2D only)
+  optional uint32 pad_w = 10[default = 0]; // The padding width (2D only)
+  optional uint32 kernel_h = 11;           // The kernel height (2D only)
+  optional uint32 kernel_w = 12;           // The kernel width (2D only)
+  optional uint32 stride_h = 13;           // The stride height (2D only)
+  optional uint32 stride_w = 14;           // The stride width (2D only)
 
   // SINGA: not supported.
   // optional uint32 group = 5 [default = 1]; // The group size for group conv
 
   optional FillerConf weight_filler = 7; // The filler for the weight
-  optional FillerConf bias_filler = 8; // The filler for the bias
+  optional FillerConf bias_filler = 8;   // The filler for the bias
   enum Engine {
-    DEFAULT = 0;
-    CAFFE = 1;
-    CUDNN = 2;
-  }
-  optional Engine engine = 15 [default = DEFAULT];
+    DEFAULT = 0; CAFFE = 1; CUDNN = 2;
+  } optional Engine engine = 15[default = DEFAULT];
 
   // The axis to interpret as "channels" when performing convolution.
   // Preceding dimensions are treated as independent inputs;
@@ -374,13 +365,12 @@ message ConvolutionConf {
   // SINGA: not supported;
   // optional bool force_nd_im2col = 17 [default = false];
 
-
   // SINGA: add by xiangrui
   // cudnn workspace size in MB
-  optional int32 workspace_byte_limit = 50 [default = 512];
+  optional int32 workspace_byte_limit = 50[default = 512];
   // cudnn algorithm preference
   // options: "fastest", "limited_workspace", "no_workspace"
-  optional string prefer = 51 [default = "fastest"];
+  optional string prefer = 51[default = "fastest"];
   // input shape
   optional int32 channels = 52;
   optional int32 height = 53;
@@ -424,7 +414,7 @@ message DataConf {
 */
 
 message DropoutConf {
-  optional float dropout_ratio = 1 [default = 0.5]; // dropout ratio
+  optional float dropout_ratio = 1[default = 0.5]; // dropout ratio
 }
 
 // DummyDataLayer fills any number of arbitrarily shaped blobs with random
@@ -448,16 +438,13 @@ message DummyDataConf {
 
 message EltwiseConf {
   enum EltwiseOp {
-    PROD = 0;
-    SUM = 1;
-    MAX = 2;
-  }
-  optional EltwiseOp operation = 1 [default = SUM]; // element-wise operation
+    PROD = 0; SUM = 1; MAX = 2;
+  } optional EltwiseOp operation = 1[default = SUM]; // element-wise operation
   repeated float coeff = 2; // blob-wise coefficient for SUM operation
 
   // Whether to use an asymptotically slower (for >2 inputs) but stabler method
   // of computing the gradient for the PROD operation. (No effect for SUM op.)
-  optional bool stable_prod_grad = 3 [default = true];
+  optional bool stable_prod_grad = 3[default = true];
 }
 
 // Message that stores hyper-parameters used by EmbedLayer
@@ -468,9 +455,9 @@ message EmbedConf {
   // 1 greater than the maximum possible input value.
   optional uint32 input_dim = 2;
 
-  optional bool bias_term = 3 [default = true]; // Whether to use a bias term
-  optional FillerConf weight_filler = 4; // The filler for the weight
-  optional FillerConf bias_filler = 5; // The filler for the bias
+  optional bool bias_term = 3[default = true]; // Whether to use a bias term
+  optional FillerConf weight_filler = 4;       // The filler for the weight
+  optional FillerConf bias_filler = 5;         // The filler for the bias
 
 }
 
@@ -479,21 +466,21 @@ message ExpConf {
   // ExpLayer computes outputs y = base ^ (shift + scale * x), for base > 0.
   // Or if base is set to the default (-1), base is set to e,
   // so y = exp(shift + scale * x).
-  optional float base = 1 [default = -1.0];
-  optional float scale = 2 [default = 1.0];
-  optional float shift = 3 [default = 0.0];
+  optional float base = 1[default = -1.0];
+  optional float scale = 2[default = 1.0];
+  optional float shift = 3[default = 0.0];
 }
 
 /// Message that stores hyper-parameters used by FlattenLayer
 message FlattenConf {
   // The first axis to flatten: all preceding axes are retained in the output.
   // May be negative to index from the end (e.g., -1 for the last axis).
-  optional int32 axis = 1 [default = 1];
+  optional int32 axis = 1[default = 1];
 
   // The last axis to flatten: all following axes are retained in the output.
   // May be negative to index from the end (e.g., the default -1 for the last
   // axis).
-  optional int32 end_axis = 2 [default = -1];
+  optional int32 end_axis = 2[default = -1];
 }
 
 /*
@@ -519,11 +506,10 @@ message HDF5OutputConf {
 
 message HingeLossConf {
   enum Norm {
-    L1 = 1;
-    L2 = 2;
+    L1 = 1; L2 = 2;
   }
-  // Specify the Norm to use L1 or L2
-  optional Norm norm = 1 [default = L1];
+      // Specify the Norm to use L1 or L2
+      optional Norm norm = 1[default = L1];
 }
 
 /*
@@ -566,29 +552,29 @@ message InfogainLossConf {
 
 message InnerProductConf {
   optional uint32 num_output = 1; // The number of outputs for the layer
-  optional bool bias_term = 2 [default = true]; // whether to have bias terms
-  optional FillerConf weight_filler = 3; // The filler for the weight
-  optional FillerConf bias_filler = 4; // The filler for the bias
+  optional bool bias_term = 2[default = true]; // whether to have bias terms
+  optional FillerConf weight_filler = 3;       // The filler for the weight
+  optional FillerConf bias_filler = 4;         // The filler for the bias
 
   // The first axis to be lumped into a single inner product computation;
   // all preceding axes are retained in the output.
   // May be negative to index from the end (e.g., -1 for the last axis).
-  optional int32 axis = 5 [default = 1];
+  optional int32 axis = 5[default = 1];
 }
 
 message DenseConf {
   optional uint32 num_output = 1; // The number of outputs for the layer
-  optional bool bias_term = 2 [default = true]; // whether to have bias terms
-  optional FillerConf weight_filler = 3; // The filler for the weight
-  optional FillerConf bias_filler = 4; // The filler for the bias
+  optional bool bias_term = 2[default = true]; // whether to have bias terms
+  optional FillerConf weight_filler = 3;       // The filler for the weight
+  optional FillerConf bias_filler = 4;         // The filler for the bias
 
   // The first axis to be lumped into a single inner product computation;
   // all preceding axes are retained in the output.
   // May be negative to index from the end (e.g., -1 for the last axis).
-  optional int32 axis = 5 [default = 1];
+  optional int32 axis = 5[default = 1];
 
   optional uint32 num_input = 20; // The number of inputs for the layer
-  optional bool transpose = 21 [default = false]; // whether transpose or not
+  optional bool transpose = 21[default = false]; // whether transpose or not
 }
 
 // Message that stores hyper-parameters used by LogLayer
@@ -596,22 +582,20 @@ message LogConf {
   // LogLayer computes outputs y = log_base(shift + scale * x), for base > 0.
   // Or if base is set to the default (-1), base is set to e,
   // so y = ln(shift + scale * x) = log_e(shift + scale * x)
-  optional float base = 1 [default = -1.0];
-  optional float scale = 2 [default = 1.0];
-  optional float shift = 3 [default = 0.0];
+  optional float base = 1[default = -1.0];
+  optional float scale = 2[default = 1.0];
+  optional float shift = 3[default = 0.0];
 }
 
 // Message that stores hyper-parameters used by LRNLayer
 message LRNConf {
-  optional uint32 local_size = 1 [default = 5];
-  optional float alpha = 2 [default = 1.];
-  optional float beta = 3 [default = 0.75];
+  optional uint32 local_size = 1[default = 5];
+  optional float alpha = 2[default = 1.];
+  optional float beta = 3[default = 0.75];
   enum NormRegion {
-    ACROSS_CHANNELS = 0;
-    WITHIN_CHANNEL = 1;
-  }
-  optional NormRegion norm_region = 4 [default = ACROSS_CHANNELS];
-  optional float k = 5 [default = 1.];
+    ACROSS_CHANNELS = 0; WITHIN_CHANNEL = 1;
+  } optional NormRegion norm_region = 4[default = ACROSS_CHANNELS];
+  optional float k = 5[default = 1.];
 }
 
 message MemoryDataConf {
@@ -623,33 +607,30 @@ message MemoryDataConf {
 
 message MVNConf {
   // This parameter can be set to false to normalize mean only
-  optional bool normalize_variance = 1 [default = true];
+  optional bool normalize_variance = 1[default = true];
 
   // This parameter can be set to true to perform DNN-like MVN
-  optional bool across_channels = 2 [default = false];
+  optional bool across_channels = 2[default = false];
 
   // Epsilon for not dividing by zero while normalizing variance
-  optional float eps = 3 [default = 1e-9];
+  optional float eps = 3[default = 1e-9];
 }
 
 message PoolingConf {
   enum PoolMethod {
-    MAX = 0;
-    AVE = 1;
-    STOCHASTIC = 2;
-  }
-  optional PoolMethod pool = 1 [default = MAX]; // The pooling method
+    MAX = 0; AVE = 1; STOCHASTIC = 2;
+  } optional PoolMethod pool = 1[default = MAX]; // The pooling method
   // Pad, kernel size, and stride are all given as a single value for equal
   // dimensions in height and width or as Y, X pairs.
-  optional uint32 pad = 4 [default = 0]; // The padding size (equal in Y, X)
-  optional uint32 pad_h = 9 [default = 0]; // The padding height
-  optional uint32 pad_w = 10 [default = 0]; // The padding width
-  optional uint32 kernel_size = 2; // The kernel size (square)
-  optional uint32 kernel_h = 5; // The kernel height
-  optional uint32 kernel_w = 6; // The kernel width
-  optional uint32 stride = 3 [default = 1]; // The stride (equal in Y, X)
-  optional uint32 stride_h = 7; // The stride height
-  optional uint32 stride_w = 8; // The stride width
+  optional uint32 pad = 4[default = 0];    // The padding size (equal in Y, X)
+  optional uint32 pad_h = 9[default = 0];  // The padding height
+  optional uint32 pad_w = 10[default = 0]; // The padding width
+  optional uint32 kernel_size = 2;         // The kernel size (square)
+  optional uint32 kernel_h = 5;            // The kernel height
+  optional uint32 kernel_w = 6;            // The kernel width
+  optional uint32 stride = 3[default = 1]; // The stride (equal in Y, X)
+  optional uint32 stride_h = 7;            // The stride height
+  optional uint32 stride_w = 8;            // The stride width
   /*
   enum Engine {
     DEFAULT = 0;
@@ -660,20 +641,20 @@ message PoolingConf {
   */
   // If global_pooling then it will pool over the size of the bottom by doing
   // kernel_h = bottom->height and kernel_w = bottom->width
-  optional bool global_pooling = 12 [default = false];
+  optional bool global_pooling = 12[default = false];
   // Shape of source
   optional int32 channels = 50;
   optional int32 height = 51;
   optional int32 width = 52;
   // whether to propagate nan
-  optional bool nan_prop = 53 [default = false];
+  optional bool nan_prop = 53[default = false];
 }
 
 message PowerConf {
   // PowerLayer computes outputs y = (shift + scale * x) ^ power.
-  optional float power = 1 [default = 1.0];
-  optional float scale = 2 [default = 1.0];
-  optional float shift = 3 [default = 0.0];
+  optional float power = 1[default = 1.0];
+  optional float scale = 2[default = 1.0];
+  optional float shift = 3[default = 0.0];
 }
 /*
 message PythonConf {
@@ -684,7 +665,8 @@ message PythonConf {
   // string, dictionary in Python dict format, JSON, etc. You may parse this
   // string in `setup` method and use it in `forward` and `backward`.
   optional string param_str = 3 [default = ''];
-  // Whether this PythonLayer is shared among worker solvers during data parallelism.
+  // Whether this PythonLayer is shared among worker solvers during data
+parallelism.
   // If true, each worker solver sequentially run forward from this layer.
   // This value should be set true if you are using it as a data layer.
   optional bool share_in_parallel = 4 [default = false];
@@ -694,13 +676,8 @@ message PythonConf {
 // Message that stores hyper-parameters used by ReductionLayer
 message ReductionConf {
   enum ReductionOp {
-    SUM = 1;
-    ASUM = 2;
-    SUMSQ = 3;
-    MEAN = 4;
-  }
-
-  optional ReductionOp operation = 1 [default = SUM]; // reduction operation
+    SUM = 1; ASUM = 2; SUMSQ = 3; MEAN = 4;
+  } optional ReductionOp operation = 1[default = SUM]; // reduction operation
 
   // The first axis to reduce to a scalar -- may be negative to index from the
   // end (e.g., -1 for the last axis).
@@ -715,9 +692,9 @@ message ReductionConf {
   // If axis == 0 (the default), the output Blob always has the empty shape
   // (count 1), performing reduction across the entire input --
   // often useful for creating new loss functions.
-  optional int32 axis = 2 [default = 0];
+  optional int32 axis = 2[default = 0];
 
-  optional float coeff = 3 [default = 1.0]; // coefficient for output
+  optional float coeff = 3[default = 1.0]; // coefficient for output
 }
 
 // Message that stores hyper-parameters used by ReLULayer
@@ -727,7 +704,7 @@ message ReLUConf {
   // Maas, A. L., Hannun, A. Y., & Ng, A. Y. (2013). Rectifier nonlinearities
   // improve neural network acoustic models. In ICML Workshop on Deep Learning
   // for Audio, Speech, and Language Processing.
-  optional float negative_slope = 1 [default = 0];
+  optional float negative_slope = 1[default = 0];
   /*
   enum Engine {
     DEFAULT = 0;
@@ -798,58 +775,50 @@ message ReshapeConf {
   //   reshape_param { shape { dim: 2  dim: 1  dim: 8  }  }
   //   reshape_param { shape { dim: 1 }  axis: 1  num_axes: 0 }
   //
-  optional int32 axis = 2 [default = 0];
-  optional int32 num_axes = 3 [default = -1];
+  optional int32 axis = 2[default = 0];
+  optional int32 num_axes = 3[default = -1];
 }
 
 message SigmoidConf {
   enum Engine {
-    DEFAULT = 0;
-    CAFFE = 1;
-    CUDNN = 2;
-  }
-  optional Engine engine = 1 [default = DEFAULT];
+    DEFAULT = 0; CAFFE = 1; CUDNN = 2;
+  } optional Engine engine = 1[default = DEFAULT];
 }
 
 message SliceConf {
   // The axis along which to slice -- may be negative to index from the end
   // (e.g., -1 for the last axis).
   // By default, SliceLayer concatenates blobs along the "channels" axis (1).
-  optional int32 axis = 3 [default = 1];
+  optional int32 axis = 3[default = 1];
   repeated uint32 slice_point = 2;
 
   // DEPRECATED: alias for "axis" -- does not support negative indexing.
-  optional uint32 slice_dim = 1 [default = 1];
+  optional uint32 slice_dim = 1[default = 1];
 }
 
-// Message that stores hyper-parameters used by SoftmaxLayer, SoftmaxWithLossLayer
+// Message that stores hyper-parameters used by SoftmaxLayer,
+// SoftmaxWithLossLayer
 message SoftmaxConf {
   enum Engine {
-    DEFAULT = 0;
-    CAFFE = 1;
-    CUDNN = 2;
-  }
-  optional Engine engine = 1 [default = DEFAULT];
+    DEFAULT = 0; CAFFE = 1; CUDNN = 2;
+  } optional Engine engine = 1[default = DEFAULT];
 
   // The axis along which to perform the softmax -- may be negative to index
   // from the end (e.g., -1 for the last axis).
   // Any other axes will be evaluated as independent softmaxes.
-  optional int32 axis = 2 [default = 1];
+  optional int32 axis = 2[default = 1];
 }
 
 message TanHConf {
   enum Engine {
-    DEFAULT = 0;
-    CAFFE = 1;
-    CUDNN = 2;
-  }
-  optional Engine engine = 1 [default = DEFAULT];
+    DEFAULT = 0; CAFFE = 1; CUDNN = 2;
+  } optional Engine engine = 1[default = DEFAULT];
 }
 
 // Message that stores hyper-parameters used by TileLayer
 message TileConf {
   // The index of the axis to tile.
-  optional int32 axis = 1 [default = 1];
+  optional int32 axis = 1[default = 1];
 
   // The number of copies (tiles) of the blob to output.
   optional int32 tiles = 2;
@@ -857,7 +826,7 @@ message TileConf {
 
 // Message that stores hyper-parameters used by ThresholdLayer
 message ThresholdConf {
-  optional float threshold = 1 [default = 0]; // Strictly positive values
+  optional float threshold = 1[default = 0]; // Strictly positive values
 }
 
 /*
@@ -897,18 +866,12 @@ message WindowDataConf {
 
 message SPPConf {
   enum PoolMethod {
-    MAX = 0;
-    AVE = 1;
-    STOCHASTIC = 2;
-  }
-  optional uint32 pyramid_height = 1;
-  optional PoolMethod pool = 2 [default = MAX]; // The pooling method
+    MAX = 0; AVE = 1; STOCHASTIC = 2;
+  } optional uint32 pyramid_height = 1;
+  optional PoolMethod pool = 2[default = MAX]; // The pooling method
   enum Engine {
-    DEFAULT = 0;
-    CAFFE = 1;
-    CUDNN = 2;
-  }
-  optional Engine engine = 6 [default = DEFAULT];
+    DEFAULT = 0; CAFFE = 1; CUDNN = 2;
+  } optional Engine engine = 6[default = DEFAULT];
 }
 
 message PReLUConf {
@@ -918,13 +881,15 @@ message PReLUConf {
   // Initial value of a_i. Default is a_i=0.25 for all i.
   optional FillerConf filler = 1;
   // Whether or not slope paramters are shared across channels.
-  optional bool channel_shared = 2 [default = false];
+  optional bool channel_shared = 2[default = false];
+  // format of the input. Default is NCHW.
+  optional string format = 50[default = "NCHW"];
 }
 
 message BatchNormConf {
   // Used in the moving average computation runningMean =
   // newMean*factor + runningMean*(1-factor).
-  optional double factor = 1 [default = 0.9];
+  optional double factor = 1[default = 0.9];
   // input shape
   optional int32 channels = 2;
   optional int32 height = 3;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/test/singa/test_flatten.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_flatten.cc b/test/singa/test_flatten.cc
new file mode 100644
index 0000000..906e4b8
--- /dev/null
+++ b/test/singa/test_flatten.cc
@@ -0,0 +1,156 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "../src/model/layer/flatten.h"
+#include "gtest/gtest.h"
+
+using singa::Flatten;
+TEST(Flatten, Setup) {
+  Flatten flt;
+  EXPECT_EQ("Flatten", flt.layer_type());
+
+  singa::LayerConf conf;
+  singa::FlattenConf *flattenconf = conf.mutable_flatten_conf();
+  flattenconf->set_axis(1);
+
+  flt.Setup(conf);
+  EXPECT_EQ(1, flt.Axis());
+}
+
+TEST(Flatten, ForwardCPU) {
+  const float x[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f,
+                      -2.f, -1.f };
+  size_t n = sizeof(x) / sizeof(float);
+  singa::Shape s = { 2, 1, 3, 2 };
+  singa::Tensor in(s);
+  in.CopyDataFromHostPtr<float>(x, n);
+
+  int axis = 3;
+  Flatten flt;
+  singa::LayerConf conf;
+  singa::FlattenConf *flattenconf = conf.mutable_flatten_conf();
+  flattenconf->set_axis(axis);
+  flt.Setup(conf);
+
+  singa::Tensor out = flt.Forward(singa::kTrain, in);
+  EXPECT_EQ(n, out.Size());
+  EXPECT_EQ(6, out.shape(0));
+  EXPECT_EQ(2, out.shape(1));
+  const float *yptr = out.data<const float *>();
+  for (size_t i = 0; i < n; i++)
+    EXPECT_FLOAT_EQ(x[i], yptr[i]);
+}
+
+TEST(Flatten, BackwardCPU) {
+  // directly use input as the output_grad for backward
+  // note that only the shape of input really matters
+  const float dy[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f,
+                       -2.f, -1.f };
+  size_t n = sizeof(dy) / sizeof(float);
+  singa::Tensor in(singa::Shape {
+    2, 1, 3, 2
+  });
+  in.CopyDataFromHostPtr<float>(dy, n);
+
+  int axis = 2;
+  Flatten flt;
+  singa::LayerConf conf;
+  singa::FlattenConf *flattenconf = conf.mutable_flatten_conf();
+  flattenconf->set_axis(axis);
+  flt.Setup(conf);
+
+  singa::Tensor temp = flt.Forward(singa::kTrain, in);
+  const auto out = flt.Backward(singa::kTrain, temp);
+  const float *xptr = out.first.data<const float *>();
+  EXPECT_EQ(n, out.first.Size());
+  EXPECT_EQ(2, out.first.shape(0));
+  EXPECT_EQ(1, out.first.shape(1));
+  EXPECT_EQ(3, out.first.shape(2));
+  EXPECT_EQ(2, out.first.shape(3));
+  for (size_t i = 0; i < n; i++)
+    EXPECT_FLOAT_EQ(dy[i], xptr[i]);
+}
+
+#ifdef USE_CUDA
+TEST(Flatten, ForwardGPU) {
+  const float x[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f,
+                      -2.f, -1.f };
+  size_t n = sizeof(x) / sizeof(float);
+  singa::CudaGPU cuda(0, 1);
+  singa::Tensor in(singa::Shape {
+    2, 1, 3, 2
+  },
+                   &cuda);
+  in.CopyDataFromHostPtr<float>(x, n);
+
+  int axis = 3;
+  Flatten flt;
+  singa::LayerConf conf;
+  singa::FlattenConf *flattenconf = conf.mutable_flatten_conf();
+  flattenconf->set_axis(axis);
+  flt.Setup(conf);
+
+  singa::Tensor out = flt.Forward(singa::kTrain, in);
+  singa::CppCPU host(0, 1);
+  out.ToDevice(&host);
+  EXPECT_EQ(n, out.Size());
+  EXPECT_EQ(6, out.shape(0));
+  EXPECT_EQ(2, out.shape(1));
+  const float *yptr = out.data<const float *>();
+  for (size_t i = 0; i < n; i++)
+    EXPECT_FLOAT_EQ(x[i], yptr[i]);
+}
+
+TEST(Flatten, BackwardGPU) {
+  // directly use input as the output_grad for backward
+  // note that only the shape of input really matters
+  const float dy[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -4.f, 1.5f, -1.5f, 0.f, -0.5f,
+                       -2.f, -1.f };
+  size_t n = sizeof(dy) / sizeof(float);
+  singa::CudaGPU cuda(0, 1);
+  singa::Tensor in(singa::Shape {
+    2, 1, 3, 2
+  },
+                   &cuda);
+  in.CopyDataFromHostPtr<float>(dy, n);
+
+  int axis = 2;
+  Flatten flt;
+  singa::LayerConf conf;
+  singa::FlattenConf *flattenconf = conf.mutable_flatten_conf();
+  flattenconf->set_axis(axis);
+  flt.Setup(conf);
+
+  singa::Tensor out = flt.Forward(singa::kTrain, in);
+  const auto ret = flt.Backward(singa::kTrain, out);
+  singa::CppCPU host(0, 1);
+  singa::Tensor in_diff = ret.first;
+  in_diff.ToDevice(&host);
+  const float *xptr = in_diff.data<const float *>();
+  EXPECT_EQ(n, in_diff.Size());
+  EXPECT_EQ(2, in_diff.shape(0));
+  EXPECT_EQ(1, in_diff.shape(1));
+  EXPECT_EQ(3, in_diff.shape(2));
+  EXPECT_EQ(2, in_diff.shape(3));
+  for (size_t i = 0; i < n; i++)
+    EXPECT_FLOAT_EQ(dy[i], xptr[i]);
+}
+#endif // USE_CUDA

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81b7/test/singa/test_prelu.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_prelu.cc b/test/singa/test_prelu.cc
new file mode 100644
index 0000000..2dde9e9
--- /dev/null
+++ b/test/singa/test_prelu.cc
@@ -0,0 +1,149 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "../src/model/layer/prelu.h"
+#include "gtest/gtest.h"
+#include "singa_config.h"
+
+using singa::PReLU;
+TEST(PReLU, Setup) {
+  PReLU prelu;
+  EXPECT_EQ("PReLU", prelu.layer_type());
+
+  singa::LayerConf conf;
+  singa::PReLUConf *preluconf = conf.mutable_prelu_conf();
+  preluconf->set_channel_shared(true);
+  preluconf->set_format("NHWC");
+
+  prelu.Setup(conf);
+  EXPECT_EQ(true, prelu.Channel_shared());
+  EXPECT_EQ("NHWC", prelu.Format());
+}
+
+TEST(PReLU, ForwardCPU) {
+  const float x[] = { 1.f, 2.f, 3.f, -2.f, -3.f, -1.f, -1.f, 2.f, -1.f, -2.f,
+                      -2.f, -1.f };
+  size_t n = sizeof(x) / sizeof(float);
+  size_t batchsize = 2, c = 3, h = 2, w = 1;
+  singa::Tensor in(singa::Shape {
+    batchsize, h, w, c
+  });
+  in.CopyDataFromHostPtr<float>(x, n);
+
+  PReLU prelu;
+  singa::LayerConf conf;
+  singa::PReLUConf *preluconf = conf.mutable_prelu_conf();
+  preluconf->set_channel_shared(false);
+  preluconf->set_format("NHWC");
+  prelu.Setup(conf);
+
+  const float neg_slope[] = { 0.25f, 0.5f, 0.75f };
+  singa::Tensor a(singa::Shape {
+    c
+  });
+  a.CopyDataFromHostPtr<float>(neg_slope, c);
+  prelu.Set_a(a);
+
+  singa::Tensor out = prelu.Forward(singa::kTrain, in);
+  const float *yptr = out.data<const float *>();
+  EXPECT_EQ(n, out.Size());
+
+  float *y = new float[n];
+  size_t div_factor = prelu.Channel_shared() ? c : 1;
+  if (prelu.Format() == "NCHW") {
+    for (size_t i = 0; i < n; i++) {
+      size_t pos = i / (h * w) % c / div_factor;
+      y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f);
+    }
+  } else if (prelu.Format() == "NHWC") {
+    for (size_t i = 0; i < n; i++) {
+      size_t pos = i % c / div_factor;
+      y[i] = std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f);
+    }
+  }
+  for (size_t i = 0; i < n; i++)
+    EXPECT_FLOAT_EQ(y[i], yptr[i]);
+}
+
+TEST(PReLU, BackwardCPU) {
+  const float x[] = {1.f, 2.f, 3.f, -2.f, -3.f, -1.f, -1.f, 2.f, -1.f, -2.f, -2.f, -1.f};
+  size_t n = sizeof(x) / sizeof(float);
+  size_t batchsize = 2, c = 3, h = 2, w = 1;
+  singa::Tensor in(singa::Shape {
+    batchsize, c, h, w
+  });
+  in.CopyDataFromHostPtr<float>(x, n);
+
+  PReLU prelu;
+  singa::LayerConf conf;
+  singa::PReLUConf *preluconf = conf.mutable_prelu_conf();
+  preluconf->set_channel_shared(false);
+  preluconf->set_format("NCHW");
+  prelu.Setup(conf);
+
+  const float neg_slope[] = { 0.25f, 0.5f, 0.75f };
+  singa::Tensor a(singa::Shape {
+    c
+  });
+  a.CopyDataFromHostPtr<float>(neg_slope, c);
+  prelu.Set_a(a);
+
+  singa::Tensor out = prelu.Forward(singa::kTrain, in);
+
+  const float grad[] = { 1.f, 2.f, -2.f, -1.f, -1.f, -3.f, 2.f, -2.f, 1.f, 1.f,
+                         -2.f, 0.f };
+  singa::Tensor out_diff(singa::Shape {
+    batchsize, c, h, w
+  });
+  out_diff.CopyDataFromHostPtr<float>(grad, n);
+  const auto ret = prelu.Backward(singa::kTrain, out_diff);
+  const float *xptr = ret.first.data<const float *>();
+  const float *aptr = ret.second.at(0).data<const float *>();
+  float *dx = new float[n];
+  size_t div_factor = prelu.Channel_shared() ? c : 1;
+  size_t params = prelu.Channel_shared() ? 1 : c;
+  float da[] = { 0.f, 0.f, 0.f };
+  if (prelu.Format() == "NCHW") {
+    for (size_t i = 0; i < n; i++) {
+      size_t pos = i / (h * w) % c / div_factor;
+      dx[i] = grad[i] *
+              (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f));
+    }
+    for (size_t i = 0; i < n; i++) {
+      size_t pos = i / (h * w) % c / div_factor;
+      da[pos] += grad[i] * std::min(x[i], 0.f);
+    }
+  } else if (prelu.Format() == "NHWC") {
+    for (size_t i = 0; i < n; i++) {
+      size_t pos = i % c / div_factor;
+      dx[i] = grad[i] *
+              (std::max(x[i], 0.f) + neg_slope[pos] * std::min(x[i], 0.f));
+    }
+    for (size_t i = 0; i < n; i++) {
+      size_t pos = i % c / div_factor;
+      da[pos] += grad[i] * std::min(x[i], 0.f);
+    }
+  }
+  for (size_t i = 0; i < n; i++)
+    EXPECT_FLOAT_EQ(dx[i], xptr[i]);
+  for (size_t i = 0; i < params; i++)
+    EXPECT_FLOAT_EQ(da[i], aptr[i]);
+}


Mime
View raw message