singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kaip...@apache.org
Subject [2/6] incubator-singa git commit: SINGA-204 Support the training of feed-forward neural nets
Date Mon, 27 Jun 2016 14:11:49 GMT
SINGA-204 Support the training of feed-forward neural nets

Fix the bug from pre/post increament in Block Inc/Dec reference which
resulted in out-of-memory.  Cifar10 data loading works.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/cf1d8418
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/cf1d8418
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/cf1d8418

Branch: refs/heads/dev
Commit: cf1d841890842c6cf1573491f4fc9d7e1eca30f4
Parents: d826b2e
Author: Wei Wang <wangwei@comp.nus.edu.sg>
Authored: Sat Jun 25 19:34:13 2016 +0800
Committer: Wei Wang <wangwei@comp.nus.edu.sg>
Committed: Mon Jun 27 15:27:19 2016 +0800

----------------------------------------------------------------------
 examples/cifar10/alexnet.cc            | 105 +++++++++++------
 examples/cifar10/cifar10.cc            |  98 ----------------
 examples/cifar10/cifar10.h             |  99 ++++++++++++++++
 examples/cifar10/make.sh               |   1 +
 include/singa/core/common.h            |  16 +--
 include/singa/core/tensor.h            |   4 +-
 include/singa/model/feed_forward_net.h |  63 +++++-----
 include/singa/model/layer.h            |   2 +-
 include/singa/model/loss.h             |   4 +-
 include/singa/model/metric.h           |  23 ++++
 include/singa/model/optimizer.h        |   2 +-
 src/CMakeLists.txt                     |   3 +-
 src/core/device/cuda_gpu.cc            |   4 +-
 src/core/tensor/tensor.cc              |  14 ++-
 src/model/feed_forward_net.cc          | 176 ++++++++++++++--------------
 src/model/layer/convolution.cc         |   6 +-
 src/model/layer/convolution.h          |   1 +
 src/model/layer/cudnn_activation.cc    |   1 +
 src/model/layer/cudnn_convolution.cc   |   1 +
 src/model/layer/cudnn_dropout.cc       |   4 +
 src/model/layer/cudnn_dropout.h        |   1 +
 src/model/layer/cudnn_pooling.cc       |   1 +
 src/model/layer/cudnn_softmax.cc       |   1 +
 src/model/layer/dense.cc               |   3 +
 src/model/metric/accuracy.cc           |  62 ++++++++++
 src/model/metric/accuracy.h            |  84 -------------
 test/singa/test_accuracy.cc            |   2 +-
 27 files changed, 419 insertions(+), 362 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/examples/cifar10/alexnet.cc
----------------------------------------------------------------------
diff --git a/examples/cifar10/alexnet.cc b/examples/cifar10/alexnet.cc
index 2917dd2..45d8571 100644
--- a/examples/cifar10/alexnet.cc
+++ b/examples/cifar10/alexnet.cc
@@ -22,7 +22,14 @@
 #include "singa/model/feed_forward_net.h"
 #include "singa/model/optimizer.h"
 #include "singa/model/initializer.h"
-
+#include "singa/model/metric.h"
+#include "singa/utils/channel.h"
+#include "singa/utils/string.h"
+#include "../../src/model/layer/cudnn_convolution.h"
+#include "../../src/model/layer/cudnn_activation.h"
+#include "../../src/model/layer/cudnn_pooling.h"
+#include "../../src/model/layer/dense.h"
+#include "../../src/model/layer/flatten.h"
 namespace singa {
 
 LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride,
@@ -32,9 +39,9 @@ LayerConf GenConvConf(string name, int nb_filter, int kernel, int stride,
   conf.set_type("CudnnConvolution");
   ConvolutionConf *conv = conf.mutable_convolution_conf();
   conv->set_num_output(nb_filter);
-  conv->set_kernel_size(kernel);
-  conv->set_stride(stride);
-  conv->set_pad(pad);
+  conv->add_kernel_size(kernel);
+  conv->add_stride(stride);
+  conv->add_pad(pad);
 
   FillerConf *weight = conv->mutable_weight_filler();
   weight->set_type("Xavier");
@@ -50,7 +57,7 @@ LayerConf GenPoolingConf(string name, bool max_pool, int kernel, int stride, int
   pool->set_stride(stride);
   pool->set_pad(pad);
   if (!max_pool)
-    pool->set_pool(PoolingConf_AVE);
+    pool->set_pool(PoolingConf_PoolMethod_AVE);
   return conf;
 }
 
@@ -65,9 +72,9 @@ LayerConf GenDenseConf(string name, int num_output) {
   LayerConf conf;
   conf.set_name(name);
   conf.set_type("Dense");
-  DenseConf *dense = conf->mutable_dense_conf();
+  DenseConf *dense = conf.mutable_dense_conf();
   dense->set_num_output(num_output);
-  FillerConf *weight = conv->mutable_weight_filler();
+  FillerConf *weight = dense->mutable_weight_filler();
   weight->set_type("Xavier");
   return conf;
 }
@@ -79,22 +86,27 @@ LayerConf GenSoftmaxConf(string name) {
   return conf;
 }
 
-
-FeedForwordNet CreateNet(Optimizer* opt, Loss* loss, Metric* metric) {
-  FeedForwordNet net;
+LayerConf GenFlattenConf(string name) {
+  LayerConf conf;
+  conf.set_name(name);
+  conf.set_type("Flatten");
+  return conf;
+}
+FeedForwardNet CreateNet(Optimizer* opt, Loss<Tensor>* loss, Metric<Tensor>* metric) {
+  FeedForwardNet net;
   Shape s{3, 32, 32};
-  net.AddLayer(GenConvConf("conv1", 32, 5, 1, 2), &s);
-  net.AddLayer(GenReLUConf("relu1"));
-  net.AddLayer(GenConvConf("pool1", 3, 2, 0));
-  net.AddLayer(GenConvConf("conv2", 32, 5, 1, 2));
-  net.AddLayer(GenReLUConf("relu2"));
-  net.AddLayer(GenConvConf("pool2", 3, 2, 0));
-  net.AddLayer(GenConvConf("conv3", 64, 5, 1, 2));
-  net.AddLayer(GenReLUConf("relu3"));
-  net.AddLayer(GenConvConf("pool3", 3, 2, 0));
-  net.AddLayer(GenDenseConf("ip1", 10));
-  net.AddLayer(GenSoftmaxConf("softmax"));
 
+  net.Add(new CudnnConvolution(), GenConvConf("conv1", 32, 5, 1, 2), &s);
+  net.Add(new CudnnActivation(), GenReLUConf("relu1"));
+  net.Add(new CudnnPooling, GenPoolingConf("pool1", true, 3, 2, 0));
+  net.Add(new CudnnConvolution(), GenConvConf("conv2", 32, 5, 1, 2));
+  net.Add(new CudnnActivation(), GenReLUConf("relu2"));
+  net.Add(new CudnnPooling(), GenPoolingConf("pool2", true, 3, 2, 0));
+  net.Add(new CudnnConvolution, GenConvConf("conv3", 64, 5, 1, 2));
+  net.Add(new CudnnActivation(), GenReLUConf("relu3"));
+  net.Add(new CudnnConvolution(), GenConvConf("pool3", true, 3, 2, 0));
+  net.Add(new Flatten(), GenFlattenConf("flat"));
+  net.Add(new Dense(), GenDenseConf("ip1", 10));
   OptimizerConf opt_conf;
   opt_conf.set_momentum(0.9);
   opt->Setup(opt_conf);
@@ -103,42 +115,57 @@ FeedForwordNet CreateNet(Optimizer* opt, Loss* loss, Metric* metric) {
 }
 
 void Train(float lr, int num_epoch, string data_dir) {
-  SoftmaxCrossEntropy loss;
-  Accuracy acc;
-  SGD sgd;
-  sgd.SetLearningRate([lr](int step) {return lr;});
-  auto net = CreateNet(&opt, &loss, &metric);
   Cifar10 data(data_dir);
-  Tensor train_x, tain_y, test_x, test_y;
+  Tensor train_x, train_y, test_x, test_y;
   {
     auto train = data.ReadTrainData();
-    const auto mean = Average(train.first, 0);
-    train_x = SubRow(train.first, mean);
-    auto test = data.ReadTestData();
-    test_x = SubRow(test.first, mean);
+    size_t nsamples = train.first.shape(0);
+    auto matx = Reshape(train.first, Shape{nsamples, train.first.Size() / nsamples});
+    const auto mean = Average(matx, 0);
+    SubRow(mean, &matx);
+    train_x = Reshape(matx, train.first.shape());
     train_y = train.second;
+    auto test = data.ReadTestData();
+    nsamples = test.first.shape(0);
+    auto maty = Reshape(test.first, Shape{nsamples, test.first.Size() / nsamples});
+    SubRow(mean, &maty);
+    test_x = Reshape(maty, test.first.shape());
     test_y = test.second;
   }
-  net.Train(100, num_epoch, train_x, train_y, test_x, test_y);
+  LOG(ERROR) << "creating net";
+  SoftmaxCrossEntropy loss;
+  Accuracy acc;
+  SGD sgd;
+  sgd.SetLearningRateGenerator([lr](int step) {return lr;});
+  auto net = CreateNet(&sgd, &loss, &acc);
+
+  auto cuda = std::make_shared<CudaGPU>();
+  net.ToDevice(cuda);
+
+  train_x.ToDevice(cuda);
+  train_y.ToDevice(cuda);
+  net.Train(50, num_epoch, train_x, train_y); // test_x, test_y);
+}
+
+
 }
 
 int main(int argc, char** argv) {
-  InitChannel();
-  int pos = ArgPos(argc, argv, "-epoch");
+  singa::InitChannel(nullptr);
+  int pos = singa::ArgPos(argc, argv, "-epoch");
   int nEpoch = 5;
   if (pos != -1)
     nEpoch = atoi(argv[pos + 1]);
-  pos = ArgPos(argc, argv, "-lr");
+  pos = singa::ArgPos(argc, argv, "-lr");
   float lr = 0.01;
   if (pos != -1)
     lr = atof(argv[pos + 1]);
-  pos = ArgPos(argc, argv, "-data");
-  string data = "cifar-10-batch-bin";
+  pos = singa::ArgPos(argc, argv, "-data");
+  string data = "cifar-10-batches-bin";
   if (pos != -1)
     data = argv[pos + 1];
 
   LOG(INFO) << "Start training";
-  Train(lr, nEpoch, data);
+  singa::Train(lr, nEpoch, data);
   LOG(INFO) << "End training";
 }
-}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/examples/cifar10/cifar10.cc
----------------------------------------------------------------------
diff --git a/examples/cifar10/cifar10.cc b/examples/cifar10/cifar10.cc
deleted file mode 100644
index 7efc18f..0000000
--- a/examples/cifar10/cifar10.cc
+++ /dev/null
@@ -1,98 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied.  See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-#include <fstream>
-#include <string>
-#include <cstdint>
-#include <iostream>
-
-using std::string;
-namespace singa {
-/// For reading cifar10 binary data as tensors.
-class Cifar10 {
- public:
-  /// 'dir_path': path to the folder including the *.bin files
-  Cifar10(string dir_path, bool normalize = true)
-      : dir_path_(dir_path), normalize_(normalize) {}
-
-  /// read all training data into an image Tensor and a label Tensor
-  const std::pair<Tensor, Tensor> ReadTrainData(bool shuffle = false);
-  /// read all test data into an image Tensor and a label Tensor
-  const std::pair<Tensor, Tensor> ReadTestData();
-  /// read data from one file into an image Tensor and a label Tensor
-  const std::pair<Tensor, Tensor> ReadFile(string file, bool shuffle = false);
-
- private:
-  const int kImageSize = 32;
-  const int kImageVol = 3072;
-  const int kBatchSize = 10000;
-  const int kTrainFiles = 5;
-
-  string dir_path_;
-  bool normalize_;
-};
-
-void read_image(std::ifstream* file, int* label, char* buffer) {
-  char label_char;
-  file->read(&label_char, 1);
-  *label = label_char;
-  file->read(buffer, kImageVol);
-  return;
-}
-const std::pair<Tensor, Tensor> Cifar10::ReadFile(string file,
-                                                  bool shuffle = false) {
-  Tensor images(Shape{kTrainFiles, 3, kImageSize, kImageSize});
-  Tensor labels(Shape{kTrainFiles}, kInt);
-  if (dir_path_.back() != '/') dir_path_.push_back('/');
-  LOG(INFO) << "Reading file " << dir_path_ + file;
-  std::ifstream data_file((dir_path_ + file).c_str(),
-                          std::ios::in | std::ios::binary);
-  CHECK(data_file.is_open()) << "Unable to open file " << file;
-  int label;
-  char image[kImageVol];
-  float float_image[kImageVol];
-  int tmplabels[kBatchSize];
-  for (int itemid = 0; itemid < kBatchSize; ++itemid) {
-    read_image(&data_file, &label, image);
-    for (int i = 0; i < kImageVol; i++)
-      float_image[i] = static_cast<float>(static_cast<int>(image[i]));
-    images.CopyDataFromHostPtr(float_image, kImageVol, itemid * kImageVol);
-    tmplabels[itemid] = label;
-  }
-  labels.CopyDataFromHostPtr(tmplabels, kBatchSize);
-  return std::make_pair(images, labels);
-}
-
-const std::pair<Tensor, Tensor> Cifar10::ReadTrainData(bool shuffle = false) {
-  Tensor images(Shape{kBatchSize * kTrainFiles, 3, kImageSize, kImageSize});
-  Tensor labels(Shape{kBatchSize * kTrainFiles, 3, kImageSize, kImageSize});
-  for (int fileid = 0; fileid < kTrainFiles; ++fileid) {
-    string file = "data_batch_" + std::to_string(fileid + 1) + ".bin";
-    const auto ret = ReadFile(file);
-    CopyDataToFrom(&images, ret.first, ret.first.Size(),
-                   fileid * ret.first.Size());
-    CopyDataToFrom(&labels, ret.second, kBatchSize, fileid * kBatchSize);
-  }
-  return std::make_pair(images, labels);
-}
-const std::pair<Tensor, Tensor> Cifar10::ReadTrainData() {
-  return ReadFile("test_batch.bin");
-}
-}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/examples/cifar10/cifar10.h
----------------------------------------------------------------------
diff --git a/examples/cifar10/cifar10.h b/examples/cifar10/cifar10.h
new file mode 100644
index 0000000..261c048
--- /dev/null
+++ b/examples/cifar10/cifar10.h
@@ -0,0 +1,99 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#include <fstream>
+#include <string>
+#include <cstdint>
+#include <iostream>
+#include "singa/core/tensor.h"
+using std::string;
+namespace singa {
+/// For reading cifar10 binary data as tensors.
+class Cifar10 {
+ public:
+  /// 'dir_path': path to the folder including the *.bin files
+  Cifar10(string dir_path, bool normalize = true)
+      : dir_path_(dir_path), normalize_(normalize) {}
+
+  /// read all training data into an image Tensor and a label Tensor
+  const std::pair<Tensor, Tensor> ReadTrainData(bool shuffle = false);
+  /// read all test data into an image Tensor and a label Tensor
+  const std::pair<Tensor, Tensor> ReadTestData();
+  /// read data from one file into an image Tensor and a label Tensor
+  const std::pair<Tensor, Tensor> ReadFile(string file, bool shuffle = false);
+
+  void ReadImage(std::ifstream* file, int* label, char* buffer);
+ private:
+  const size_t kImageSize = 32;
+  const size_t kImageVol = 3072;
+  const size_t kBatchSize = 10000;
+  const size_t kTrainFiles = 1;
+
+  string dir_path_;
+  bool normalize_;
+};
+
+void Cifar10::ReadImage(std::ifstream* file, int* label, char* buffer) {
+  char label_char;
+  file->read(&label_char, 1);
+  *label = static_cast<int>(label_char);
+  file->read(buffer, kImageVol);
+  return;
+}
+const std::pair<Tensor, Tensor> Cifar10::ReadFile(string file, bool shuffle) {
+  Tensor images(Shape{kBatchSize, 3, kImageSize, kImageSize});
+  Tensor labels(Shape{kBatchSize}, kInt);
+  if (dir_path_.back() != '/') dir_path_.push_back('/');
+  LOG(INFO) << "Reading file " << dir_path_ + file;
+  std::ifstream data_file((dir_path_ + file).c_str(),
+                          std::ios::in | std::ios::binary);
+  CHECK(data_file.is_open()) << "Unable to open file " << dir_path_ + file;
+  int label;
+  char image[kImageVol];
+  float float_image[kImageVol];
+  int tmplabels[kBatchSize];
+  for (int itemid = 0; itemid < kBatchSize; ++itemid) {
+    // LOG(INFO) << "reading " << itemid << "-th image";
+    ReadImage(&data_file, &label, image);
+    for (int i = 0; i < kImageVol; i++)
+      float_image[i] = static_cast<float>(static_cast<int>(image[i]));
+    images.CopyDataFromHostPtr(float_image, kImageVol, itemid * kImageVol);
+    tmplabels[itemid] = label;
+  }
+  labels.CopyDataFromHostPtr(tmplabels, kBatchSize);
+  return std::make_pair(images, labels);
+}
+
+const std::pair<Tensor, Tensor> Cifar10::ReadTrainData(bool shuffle) {
+  Tensor images(Shape{kBatchSize * kTrainFiles, 3, kImageSize, kImageSize});
+  Tensor labels(Shape{kBatchSize * kTrainFiles}, kInt);
+  for (int fileid = 0; fileid < kTrainFiles; ++fileid) {
+    string file = "data_batch_" + std::to_string(fileid + 1) + ".bin";
+    const auto ret = ReadFile(file);
+    CopyDataToFrom(&images, ret.first, ret.first.Size(),
+                   fileid * ret.first.Size());
+    CopyDataToFrom(&labels, ret.second, kBatchSize, fileid * kBatchSize);
+  }
+  return std::make_pair(images, labels);
+}
+const std::pair<Tensor, Tensor> Cifar10::ReadTestData() {
+  return ReadFile("test_batch.bin");
+}
+}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/examples/cifar10/make.sh
----------------------------------------------------------------------
diff --git a/examples/cifar10/make.sh b/examples/cifar10/make.sh
new file mode 100755
index 0000000..17e4b39
--- /dev/null
+++ b/examples/cifar10/make.sh
@@ -0,0 +1 @@
+g++ -g --std=c++11 alexnet.cc -o alexnet -I../../include -I../../build/include -I/home/wangwei/local/cudnn4/include -I/home/wangwei/local/include -I/usr/local/cuda/include/ -I../../lib/cnmem/include -L../../build/lib/ -lsinga_core -lsinga_model  -lsinga_utils -lcudart -lcublas -lcurand -lcudnn -L/usr/local/cuda/lib64 -L/home/wangwei/local/cudnn4/lib64 ../../build/lib/libproto.a -lprotobuf

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/include/singa/core/common.h
----------------------------------------------------------------------
diff --git a/include/singa/core/common.h b/include/singa/core/common.h
index cb1bdca..691d7d4 100644
--- a/include/singa/core/common.h
+++ b/include/singa/core/common.h
@@ -49,27 +49,29 @@ class Block {
  public:
   Block(void* ptr, size_t size, size_t offset = 0)
       : data_(ptr), size_(size), offset_(offset) {
-    ref_count_ = std::make_shared<std::atomic<int>>(1);
+    ref_count_ = 1; //std::make_shared<std::atomic<int>>(1);
   }
-  Block(void* ptr, size_t size, size_t offset, std::shared_ptr<atomic<int>> ref)
-      : data_(ptr), size_(size), offset_(offset), ref_count_(ref) {}
+//  Block(void* ptr, size_t size, size_t offset, std::shared_ptr<atomic<int>> ref)
+//      : data_(ptr), size_(size), offset_(offset), ref_count_(ref) {}
   void* mutable_data() const { return static_cast<char*>(data_) + offset_; }
   const void* data() const { return static_cast<char*>(data_) + offset_; }
   size_t size() const { return size_; }
   size_t offset() const { return offset_; }
   int IncRefCount() {
-    return (*ref_count_)++;
+    return ++ref_count_;  //(*ref_count_)++;
   }
   int DecRefCount() {
-    return  (*ref_count_)--;
+    return --ref_count_; // (*ref_count_)--;
   }
-  int ref_count() const { return ref_count_->load(); }
+  int ref_count() const { return ref_count_.load(); }
 
  private:
+  Block() {}
   void* data_ = nullptr;
   size_t size_ = 0;
   size_t offset_ = 0;
-  std::shared_ptr<std::atomic<int>> ref_count_ = nullptr;
+  // std::shared_ptr<std::atomic<int>> ref_count_ = nullptr;
+  std::atomic<int> ref_count_;
 };
 
 typedef struct _Context {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/include/singa/core/tensor.h
----------------------------------------------------------------------
diff --git a/include/singa/core/tensor.h b/include/singa/core/tensor.h
index 6de5c0c..3b496d9 100644
--- a/include/singa/core/tensor.h
+++ b/include/singa/core/tensor.h
@@ -96,6 +96,8 @@ class Tensor {
 
   /// return number of total elements
   size_t Size() const {
+    if (block_ == nullptr)
+      return 0u;
     CHECK_EQ(block_->size() % SizeOf(data_type_), 0u);
     return block_->size() / SizeOf(data_type_);
   }
@@ -315,7 +317,7 @@ Tensor Div(const SType x, const Tensor &in);
 template <typename SType>
 void Div(const SType x, const Tensor &in, Tensor *out);
 
-template <typename SType>
+template <typename SType = float>
 SType Sum(const Tensor &in);
 // ============Matrix (row/column) operations==================================
 /// Average elements in the Tensor, currently only support vector and matrix.

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/include/singa/model/feed_forward_net.h
----------------------------------------------------------------------
diff --git a/include/singa/model/feed_forward_net.h b/include/singa/model/feed_forward_net.h
index 173600b..9beeb7a 100644
--- a/include/singa/model/feed_forward_net.h
+++ b/include/singa/model/feed_forward_net.h
@@ -18,7 +18,9 @@
 #ifndef SINGA_MODEL_FEED_FORWARD_NET_H_
 #define SINGA_MODEL_FEED_FORWARD_NET_H_
 #include "singa/model/layer.h"
-
+#include "singa/model/loss.h"
+#include "singa/model/metric.h"
+#include "singa/model/optimizer.h"
 namespace singa {
 
 /// The feed-forward neural net.
@@ -26,14 +28,14 @@ namespace singa {
 /// and conducting training, evaluation and prediction.
 class FeedForwardNet {
  public:
-  FeedForwardNet() = explicit;
+  FeedForwardNet() = default;
   ~FeedForwardNet();
 
   /// Add a layer with the assumption that
   /// 1. this function is called in correct order, i.e., the layers are added
   ///    following the topological order.
   /// 2. this layer has already been setup (Setup function is called outside).
-  void Add(Layer *layer);
+  Layer* Add(Layer* layer);
 
   // TODO(wangwei) add ConcatenateLayer and SliceLayer
   // AddConcatenateLayer(vector<Layer*> src, Layer *dst);
@@ -43,36 +45,34 @@ class FeedForwardNet {
   /// Assume the layer is added in corret order.
   /// For the first layer, 'sample_shape' (the input sample shape) is necessary
   /// for calling Setup().
-  void Add(const LayerConf &conf, const Shape *sample_shape = nullptr);
+  Layer* Add(const LayerConf& conf, const Shape* sample_shape = nullptr);
 
+  Layer* Add(Layer* layer, const LayerConf& conf, const Shape* sample_shape = nullptr);
   /// Set some fields used for training and evaluating the neural net.
   /// If the neural net is constructed for evaluation only, then 'opt' is not
   /// necessary; But for training, both 'opt' and 'loss' are necessary.
   /// 'shuffle' indicates shuffling training samples within one epoch it is
   /// valid using Train();
-  void Compile(bool shuffle, Optimizer *opt, Loss *loss, Metric *metric);
+  void Compile(bool shuffle, Optimizer* opt, Loss<Tensor>* loss,
+               Metric<Tensor>* metric);
 
   /// Conduct the training giving the training data 'x' and label 'y'.
-  /// Due to memory limit, 'x' and 'y' could not be very large. Hence, it is
-  /// typically used for small training datasets, e.g., cifar10 and MNIST which
-  /// can be stored in main memory.
-  void Train(int batchsize, int nb_epoch, Tensor x, Tensor y);
-  /// Conduct the training giving the training data 'x' and label 'y'.
-  /// 'val_split' is a ratio for splitting (1-'val_split') of training data for
+  /// 'val_split' of training data is used for
   /// validation. Validation is performance before every epoch.
   /// Due to memory limit, 'x' and 'y' could not be very large. Hence, it is
   /// typically used for small training datasets, e.g., cifar10 and MNIST which
   /// can be stored in main memory.
-  void Train(int batchsize, int nb_epoch, float val_split, Tensor x, Tensor y);
+  void Train(size_t batchsize, int nb_epoch, const Tensor& x, const Tensor& y,
+             float val_split = 0.0f);
   /// Conduct the training given the training and validation data.
   /// Validation is performance before every epoch.
   /// Due to memory limit, 'x' and 'y' could not be very large. Hence, it is
   /// typically used for small training datasets, e.g., cifar10 and MNIST which
   /// can be stored in main memory.
-  void Train(int batchsize, int nb_epoch, Tensor x, Tensor y, Tensor val_x,
-             Tensor val_y);
+  void Train(size_t batchsize, int nb_epoch, const Tensor& x, const Tensor& y,
+             const Tensor& val_x, const Tensor& val_y);
   /// Train the neural net over one batch of training data.
-  Tensor TrainOnBatch(Tensor x, Tensor y);
+  const std::pair<float, float> TrainOnBatch(const Tensor& x, const Tensor& y);
 
   /// Evaluate the neural net with given data.
   /// Returns one tensor for loss values and one tensor for metric values;
@@ -82,9 +82,10 @@ class FeedForwardNet {
   /// Due to memory limit, 'x' and 'y' could not be very large. Hence, it is
   /// typically used for small training datasets, e.g., cifar10 and MNIST which
   /// can be stored in main memory.
-  std::pair<Tensor, Tensor> Evaluate(Tensor x, Tensor y, int batchsize = 128);
+  std::pair<Tensor, Tensor> Evaluate(const Tensor& x, const Tensor& y,
+                                     size_t batchsize = 128);
   /// Evaluate the neural net for one batch of data
-  std::pair<Tensor, Tensor> EvaluateOnBatch(Tensor x, Tensor y);
+  std::pair<Tensor, Tensor> EvaluateOnBatch(const Tensor& x, const Tensor& y);
 
   /// Predict the probability distributation over candicate classes for each
   /// data sample. 'batchsize' is used for controlling the memory footprint.
@@ -92,35 +93,37 @@ class FeedForwardNet {
   /// Due to memory limit, 'x' and 'y' could not be very large. Hence, it is
   /// typically used for small training datasets, e.g., cifar10 and MNIST which
   /// can be stored in main memory.
-  Tensor Predict(const Tensor &x, int batchsize = 128);
+  const Tensor Predict(const Tensor& x, size_t batchsize = 128);
   /// Predict for one batch data.
-  Tensor PredictOnBatch(const Tensor &x);
+  const Tensor PredictOnBatch(const Tensor& x);
 
   /// Forward layers one by one using the data batch 'x'.
   /// Returns the prediction results (from the last layer).
-  Tensor Forward(const Tensor& x);
+  const Tensor Forward(int flag, const Tensor& x);
   /// Backward layers one by one using the gradient batch 'grad'.
   /// Returns the parameter gradients.
-  const vector<Tensor> Backward(const Tensor& grad);
+  const vector<Tensor> Backward(int flag, const Tensor& grad);
 
   /// Clone the neuaral net by cloning every layer to the given device.
   /// If 'device' is nullptr, then clone it one the current device.
-  FeedForwardNet Clone(std::shared_ptr<Device> device = nullptr);
+  FeedForwardNet Clone(std::shared_ptr<Device> device);
   /// Move the layer data to the given device.
-  void ToDevice(Device *device);
+  void ToDevice(std::shared_ptr<Device> device);
+  void ToHost() { ToDevice(defaultDevice); }
   /// Set the data type of each layer.
   void AsType(DataType dtype);
 
-  const vector<Layer *> layers() const { return layers_; }
+  const vector<Layer*> layers() const { return layers_; }
   const vector<string> GetParamNames() const;
-  const vector<Tensor *> GetParamValues() const;
-  const vector<Tensor *> GetParamGrads() const;
+  const vector<ParamSpec> GetParamSpecs() const;
+  const vector<Tensor*> GetParamValues() const;
+  const vector<Tensor*> GetParamGrads() const;
 
  protected:
-  vector<Layer *> layers_;
-  Optimizer *opt_;
-  Loss *loss_;
-  Metric *metric_;
+  vector<Layer*> layers_;
+  Optimizer* opt_;
+  Loss<Tensor>* loss_;
+  Metric<Tensor>* metric_;
 
   bool shuffle_ = true;
   Device* device_ = nullptr;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/include/singa/model/layer.h
----------------------------------------------------------------------
diff --git a/include/singa/model/layer.h b/include/singa/model/layer.h
index 79eb069..ce8007c 100644
--- a/include/singa/model/layer.h
+++ b/include/singa/model/layer.h
@@ -151,7 +151,7 @@ class Layer {
 
   /// Clone the layer to the given device. Layer data (e.g., parameters) are
   /// deep copied. If 'device' is nullptr, then clone it one the current device.
-  virtual Layer* Clone(std::shared_ptr<Device> device);
+  // virtual Layer* Clone(std::shared_ptr<Device> device);
   /// Move the layer (including its parameters and other internal Tensor) onto
   /// the given device
   virtual void ToDevice(std::shared_ptr<Device> device) {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/include/singa/model/loss.h
----------------------------------------------------------------------
diff --git a/include/singa/model/loss.h b/include/singa/model/loss.h
index 79abace..41ec701 100644
--- a/include/singa/model/loss.h
+++ b/include/singa/model/loss.h
@@ -37,6 +37,7 @@ class Loss {
     Setup(loss);
   }
 	virtual ~Loss(){};
+  virtual void ToDevice(std::shared_ptr<Device> device) {}
   /// Set meta fields from user configurations.
   virtual void Setup(const LossConf& conf) {}
 
@@ -48,7 +49,8 @@ class Loss {
   /// It calls Forward() internally. The calling pattern should be
   /// [Evaluate|Forward] Backward.
   float Evaluate(const Tensor& prediction, const T& target) {
-    const Tensor& loss = Forward(prediction, target);
+    Tensor loss = Forward(prediction, target);
+    loss.ToHost();
     return Sum<float>(loss) / (1.0f * loss.Size());
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/include/singa/model/metric.h
----------------------------------------------------------------------
diff --git a/include/singa/model/metric.h b/include/singa/model/metric.h
index b99ff0d..d013fa4 100644
--- a/include/singa/model/metric.h
+++ b/include/singa/model/metric.h
@@ -33,6 +33,7 @@ class Metric {
  public:
   // TODO(wangwei) call Setup using a default MetricConf.
   Metric() = default;
+  virtual void ToDevice(std::shared_ptr<Device> device) {}
   void Setup(const string& conf) {
     MetricConf metric;
     metric.ParseFromString(conf);
@@ -51,6 +52,28 @@ class Metric {
     return Sum<float>(metric) / (1.0f * metric.Size());
   }
 };
+/// Compute the accuray of the prediction, which is matched against the
+/// ground truth labels.
+/// TODO(wangwei) consider multi-label cases.
+class Accuracy : public Metric<Tensor> {
+ public:
+  /// Set meta fields from user configurations.
+  void Setup(const MetricConf& conf) override { top_k_ = conf.top_k(); }
+
+  /// Check the prediction against the target (ground truth) for each data
+  /// sample. The returned Tensor has a float value for each sample, 0 for wrong
+  /// and 1 for correct. Users can call Sum(const Tensor&) / Tensor::Size() to
+  /// get the accuracy.
+  Tensor Forward(const Tensor& prediction, const Tensor& target);
+
+ private:
+  /// \copydoc Match(const Tensor&, const Tensor&);
+  Tensor Match(const Tensor& prediction, const vector<int>& target);
+  /// If the ground truth label is in the top k predicted labels, then the
+  /// prediction is correct.
+  size_t top_k_ = 1;
+};
+
 
 }  // namespace singa
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/include/singa/model/optimizer.h
----------------------------------------------------------------------
diff --git a/include/singa/model/optimizer.h b/include/singa/model/optimizer.h
index f912668..a268126 100644
--- a/include/singa/model/optimizer.h
+++ b/include/singa/model/optimizer.h
@@ -155,7 +155,7 @@ class Regularizer {
 };
 
 // =============Vallina SGD with Momentum=====================================
-class SGD : Optimizer {
+class SGD : public Optimizer {
  public:
   void Setup(const OptimizerConf& conf);
   /// Apply the updating algorithm.

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c51d454..af09799 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -36,7 +36,7 @@ IF (USE_CUDA)
     SET(FLAGS_BACKUP ${CMAKE_CXX_FLAGS})
     SET(CMAKE_CXX_FLAGS "")
     IF (CMAKE_BUILD_TYPE MATCHES DEBUG)
-        CUDA_COMPILE(cuda_objs SHARED ${cuda_source} 
+        CUDA_COMPILE(cuda_objs SHARED ${cuda_source}
             OPTIONS "-Xcompiler -fPIC -G -g")
     ELSE (CMAKE_BUILD_TYPE MATCHES  DEBUG)
         CUDA_COMPILE(cuda_objs SHARED ${cuda_source} OPTIONS "-Xcompiler -fPIC")
@@ -57,6 +57,7 @@ AUX_SOURCE_DIRECTORY(model model_source)
 AUX_SOURCE_DIRECTORY(model/layer model_source)
 AUX_SOURCE_DIRECTORY(model/optimizer model_source)
 AUX_SOURCE_DIRECTORY(model/loss model_source)
+AUX_SOURCE_DIRECTORY(model/metric model_source)
 #MESSAGE(STATUS "MODEL ${model_source}")
 ADD_LIBRARY(singa_model SHARED ${model_source})
 TARGET_LINK_LIBRARIES(singa_model ${SINGA_LINKER_LIBS})

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/core/device/cuda_gpu.cc
----------------------------------------------------------------------
diff --git a/src/core/device/cuda_gpu.cc b/src/core/device/cuda_gpu.cc
index 5879c58..5f6ac17 100644
--- a/src/core/device/cuda_gpu.cc
+++ b/src/core/device/cuda_gpu.cc
@@ -121,7 +121,7 @@ void CudaGPU::CopyToFrom(void* dst, const void* src, size_t nBytes,
   // cudaMemcpyAsync(dst, src, nBytes,cudaMemcpyDefault, ctx_.stream);
 }
 
-/// Allocate cpu memory.
+/// Allocate gpu memory.
 void* CudaGPU::Malloc(int size) {
   void* ptr = nullptr;
   if (size > 0) {
@@ -132,7 +132,7 @@ void* CudaGPU::Malloc(int size) {
   return ptr;
 }
 
-/// Free cpu memory.
+/// Free gpu memory.
 void CudaGPU::Free(void* ptr) {
   if (ptr != nullptr) {
     // CUDA_CHECK(cudaFree(ptr));

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/core/tensor/tensor.cc
----------------------------------------------------------------------
diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc
index ec59aaa..898cdc6 100644
--- a/src/core/tensor/tensor.cc
+++ b/src/core/tensor/tensor.cc
@@ -118,7 +118,8 @@ void Tensor::ToDevice(std::shared_ptr<Device> dst) {
   // TODO(wangwei) the comparison is very strict. May compare against device ID?
   if (device_ != dst) {
     Tensor tmp(shape_, dst, data_type_);
-    tmp.CopyData(*this);
+    if (block_ != nullptr && Size())
+      tmp.CopyData(*this);
     if (block_ != nullptr && block_->DecRefCount() == 0)
       device_->FreeBlock(block_);
     block_ = tmp.block_;
@@ -136,7 +137,8 @@ void Tensor::CopyDataFromHostPtr(const DType *src, const size_t num,
       << "data_type is " << DataType_Name(data_type_)
       << " user given type is of size " << sizeof(DType);
   if (src != nullptr) {
-    device_->CopyDataFromHostPtr(block(), src, sizeof(DType) * num, offset);
+    device_->CopyDataFromHostPtr(block(), src, sizeof(DType) * num,
+        sizeof(DType) * offset);
   } else {
     LOG(WARNING) << "Copy data from null host ptr";
   }
@@ -637,13 +639,13 @@ Tensor ConcatenateColumns(const vector<Tensor> &in) {
   return out;
 }
 Tensor CopyRows(const Tensor &in, const size_t start, const size_t end) {
-  CHECK_EQ(in.nDim(), 2u);
   CHECK_LT(start, end);
   CHECK_GE(in.shape(0), end);
-  Shape s;
-  s = Shape{end - start, in.shape(1)};
+  Shape s = in.shape();
+  s[0] = end - start;
+  size_t sample_size = in.Size() / in.shape(0);
   Tensor out(s, in.device(), in.data_type());
-  CopyDataToFrom(&out, in, out.Size(), 0, start * out.shape(1));
+  CopyDataToFrom(&out, in, out.Size(), 0, start * sample_size);
   return out;
 }
 Tensor CopyColumns(const Tensor &in, const size_t start, const size_t end) {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/feed_forward_net.cc
----------------------------------------------------------------------
diff --git a/src/model/feed_forward_net.cc b/src/model/feed_forward_net.cc
index f9e6480..a24d36a 100644
--- a/src/model/feed_forward_net.cc
+++ b/src/model/feed_forward_net.cc
@@ -18,11 +18,11 @@
 
 #include "singa/model/feed_forward_net.h"
 #include "singa/utils/logging.h"
+#include "singa/utils/channel.h"
 namespace singa {
 
-~FeedForwardNet::FeedForwardNet() {
-  for (auto layer : layers_)
-    delete layer;
+FeedForwardNet::~FeedForwardNet() {
+  for (auto layer : layers_) delete layer;
 }
 Layer* FeedForwardNet::Add(Layer* layer) {
   layers_.push_back(layer);
@@ -32,7 +32,12 @@ Layer* FeedForwardNet::Add(Layer* layer) {
 Layer* FeedForwardNet::Add(const LayerConf& conf, const Shape* sample_shape) {
   CHECK(sample_shape != nullptr || layers_.size())
       << "Must provide the input sample shape for the first layer";
-  Layer* layer = CreateLayer(conf.type());
+  Layer* layer = nullptr;  // TODO(wangwei) use CreateLayer(conf.type());
+  Add(layer, conf, sample_shape);
+  return layer;
+}
+
+Layer* FeedForwardNet::Add(Layer* layer, const LayerConf& conf, const Shape* sample_shape) {
   if (sample_shape == nullptr)
     layer->Setup(layers_.back()->GetOutputSampleShape(), conf);
   else
@@ -44,28 +49,25 @@ Layer* FeedForwardNet::Add(const LayerConf& conf, const Shape* sample_shape) {
 const vector<string> FeedForwardNet::GetParamNames() const {
   vector<string> names;
   for (auto layer : layers_)
-    for (const auto name : layer->param_names())
-      names.push_back(name);
+    for (const auto name : layer->param_names()) names.push_back(name);
   return names;
 }
-const vector<Tensor *> FeedForwardNet::GetParamValues() const {
-  vector<Tensor *> values;
+const vector<Tensor*> FeedForwardNet::GetParamValues() const {
+  vector<Tensor*> values;
   for (auto layer : layers_)
-    for (const auto value : layer->param_values())
-      values.push_back(value);
+    for (const auto value : layer->param_values()) values.push_back(value);
   return values;
 }
 
-const vector<Tensor *> FeedForwardNet::GetParamSpecs() const {
-  vector<ParamSpec *> specs;
+const vector<ParamSpec> FeedForwardNet::GetParamSpecs() const {
+  vector<ParamSpec> specs;
   for (auto layer : layers_)
-    for (const auto spec : layer->param_specs())
-      specs.push_back(spec);
+    for (const auto spec : layer->param_specs()) specs.push_back(spec);
   return specs;
 }
 
-void FeedForwardNet::Compile(bool shuffle, Optimizer* opt, Loss* loss,
-                             Metric* metric) {
+void FeedForwardNet::Compile(bool shuffle, Optimizer* opt, Loss<Tensor>* loss,
+                             Metric<Tensor>* metric) {
   shuffle_ = shuffle;
   bool train = (opt != nullptr) && (loss != nullptr);
   bool test = metric != nullptr;
@@ -73,14 +75,17 @@ void FeedForwardNet::Compile(bool shuffle, Optimizer* opt, Loss* loss,
   opt_ = opt;
   loss_ = loss;
   metric_ = metric;
+  // init params and register them to sgd
 }
 
 void FeedForwardNet::ToDevice(std::shared_ptr<Device> device) {
   for (auto layer: layers_)
     layer->ToDevice(device);
+  /*
   opt_->ToDevice(device);
   loss_->ToDevice(device);
   metric_->ToDevice(device);
+  */
 }
 
 FeedForwardNet FeedForwardNet::Clone(std::shared_ptr<Device> device) {
@@ -98,118 +103,110 @@ FeedForwardNet FeedForwardNet::Clone(std::shared_ptr<Device> device) {
   net.device_ = device;
   net.dtype_ = dtype;
   */
+  return net;
 }
 
 void FeedForwardNet::AsType(DataType dtype) {
   LOG(FATAL) << "FeedForwardNet::AsType not implemented";
 }
 
-void FeedForwardNet::Train(int batchsize, int nb_epoch, Tensor x, Tensor y) {
-  CHECK_EQ(x.shape(0), y.shape(0)) << "Diff num of sampels in x and y";
-  int num_extra_samples = x.shape(0) % batchsize;
-  if (num_extra_samples != 0)
-    LOG(WARNING) << "The last " << num_extra_samples << " would not be used";
-  Channel *ch = GetChannel("perf");
-  for (int epoch = 0; epoch < nb_epoch; epoch++) {
-    float loss = 0.0f, metric = 0.0f;
-    int batch = 0;
-    for (; batch < x.shape(0) / batchsize; batch++) {
-      Tesnor bx = x.Slice(batch * batchsize, batch * batchsize + batchsize);
-      Tesnor by = y.Slice(batch * batchsize, batch * batchsize + batchsize);
-      const auto ret = TrainOnBatch(bx, by);
-      loss += ret.first;
-      metric += ret.second;
-    }
-    loss /= batch;
-    metric /= batch;
-    ch->Send("Epoch " + std::to_string(epoch) + ", training loss = " +
-             std::to_string(loss) + ", accuracy = " + std::to_string(metric));
-  }
-}
-
-void FeedForwardNet::Train(int batchsize, int nb_epoch, Tensor x, Tensor y,
-   float val_split) {
+void FeedForwardNet::Train(size_t batchsize, int nb_epoch, const Tensor& x,
+                           const Tensor& y, float val_split) {
   CHECK_EQ(x.shape(0), y.shape(0)) << "Diff num of sampels in x and y";
   size_t num_train = x.shape(0) * val_split;
-  const Tensor train_x = CopyRows(x, 0, num_train);
-  const Tensor train_y = CopyRows(y, 0, num_train);
-  const Tensor val_x = CopyRows(x, num_train, x.shape(0));
-  const Tensor val_y = CopyRows(y, num_train, x.shape(0));
-  Train(batchsize, nb_epoch, train_x, train_y, val_x, val_y);
+  if (val_split == 0.0f) {
+    Tensor dummy;
+    Train(batchsize, nb_epoch, x, y, dummy, dummy);
+  } else {
+    const Tensor train_x = CopyRows(x, 0, num_train);
+    const Tensor train_y = CopyRows(y, 0, num_train);
+    const Tensor test_x = CopyRows(x, num_train, x.shape(0));
+    const Tensor test_y = CopyRows(y, num_train, y.shape(0));
+    Train(batchsize, nb_epoch, train_x, train_y, test_x, test_y);
+  }
 }
 
-
-void FeedForwardNet::Train(int batchsize, int nb_epoch, Tensor x, Tensor y,
-    const Tensor & val_x, const Tensor &val_y) {
+void FeedForwardNet::Train(size_t batchsize, int nb_epoch, const Tensor& x,
+                           const Tensor& y, const Tensor& val_x,
+                           const Tensor& val_y) {
+  InitNetParams();
   CHECK_EQ(x.shape(0), y.shape(0)) << "Diff num of sampels in x and y";
   int num_extra_samples = x.shape(0) % batchsize;
   if (num_extra_samples != 0)
     LOG(WARNING) << "The last " << num_extra_samples << " would not be used";
-  Channel *train_ch = GetChannel("train_perf");
-  Channel *test_ch = GetChannel("test_perf");
+  Channel* train_ch = GetChannel("train_perf");
+  train_ch->EnableDestStderr(true);
+  Channel* val_ch = GetChannel("val_perf");
   for (int epoch = 0; epoch < nb_epoch; epoch++) {
     float loss = 0.0f, metric = 0.0f;
-    int b = 0;
-    for (;b < x.shape(0) / batchsize; b++) {
-      Tesnor bx = CopyRows(x, b * batchsize, b * batchsize + batchsize);
-      Tesnor by = CopyRows(y, b * batchsize, b * batchsize + batchsize);
+    size_t b = 0;
+    for (; b < x.shape(0) / batchsize; b++) {
+      const Tensor bx = CopyRows(x, b * batchsize, b * batchsize + batchsize);
+      const Tensor by = CopyRows(y, b * batchsize, b * batchsize + batchsize);
       const auto ret = TrainOnBatch(bx, by);
       loss += ret.first;
       metric += ret.second;
     }
-    loss /= batch;
-    metric /= batch;
+    loss /= b;
+    metric /= b;
     train_ch->Send("Epoch " + std::to_string(epoch) + ", training loss = " +
-             std::to_string(loss) + ", accuracy = " + std::to_string(metric));
-    const auto val_perf = Evaluate(val_x, val_y, batchsize);
-    test_ch->Send("Epoch " + std::to_string(epoch)
-        + ", test loss = " + std::to_string(Average(val_perf.first))
-        + ", metric = " + std::to_string(Average(val_perf.second)));
+                   std::to_string(loss) + ", accuracy = " +
+                   std::to_string(metric));
+    if (val_x.Size() && val_y.Size()) {
+      const auto val_perf = Evaluate(val_x, val_y, batchsize);
+      val_ch->Send("Epoch " + std::to_string(epoch) + ", val loss = " +
+                   std::to_string(Sum(val_perf.first) / val_y.Size()) +
+                   ", metric = " +
+                   std::to_string(Sum(val_perf.second) / val_y.Size()));
+    }
   }
 }
 
-const std::pair<float, float> FeedForwardNet::TrainOnBatch(const Tensor x,
-                                                           const Tensor y) {
-  const Tensor fea = Forward(kTrain, bx);
-  float loss = loss->Evaluate(fea, fy);
-  float metric = metric->Evaluate(fea, by);
-  const Tensor grad = loss->Backward();
-  Backward(kTrain, grad);
+const std::pair<float, float> FeedForwardNet::TrainOnBatch(const Tensor& x,
+                                                           const Tensor& y) {
+  int flag = kTrain;
+  const Tensor fea = Forward(flag, x);
+  float loss = loss_->Evaluate(fea, y);
+  float metric = metric_->Evaluate(fea, y);
+  const Tensor grad = loss_->Backward();
+  const auto grads = Backward(kTrain, grad);
   return std::make_pair(loss, metric);
 }
 
-const Tensor FeedForwardNet::Forward(int flag, const Tensor data) {
-  Tensor tmp = data;
+const Tensor FeedForwardNet::Forward(int flag, const Tensor& data) {
+  Tensor input = data, output;
   for (auto layer : layers_) {
-    tmp = layer->Forward(flag, tmp);
+//    LOG(INFO) << layer->name();
+    output = layer->Forward(flag, input);
+    input = output;
   }
-  return tmp;
+  return output;
 }
 
-cons vector<Tensor> FeedForwardNet::Backward(int flag, const Tensor grad) {
+const vector<Tensor> FeedForwardNet::Backward(int flag, const Tensor& grad) {
   vector<Tensor> param_grads;
   Tensor tmp = grad;
-  for (size_t i = layers_.size() - 1; i >= 0; i--) {
+  for (int i = layers_.size() - 1; i >= 0; i--) {
+ //   LOG(INFO) << layers_.at(i)->name();
     auto ret = layers_.at(i)->Backward(flag, tmp);
-    tmp =ret.first;
+    tmp = ret.first;
     if (ret.second.size())
-      for (const auto x: ret.second)
-        param_grads.push_back(x);
+      for (const auto x : ret.second) param_grads.push_back(x);
   }
   return param_grads;
 }
 
-std::pair<Tensor, Tensor> Evaluate(Tensor x, Tensor y, int batchsize) {
+std::pair<Tensor, Tensor> FeedForwardNet::Evaluate(const Tensor& x,
+                                                   const Tensor& y,
+                                                   size_t batchsize) {
   CHECK_EQ(x.shape(0), y.shape(0)) << "Diff num of sampels in x and y";
   CHECK_GE(x.shape(0), batchsize);
   int num_extra_samples = x.shape(0) % batchsize;
-  int b = 0;
   Tensor loss(Shape{x.shape(0)}), metric(Shape{x.shape(0)});
-  for (; b < x.shape(0) / batchsize; b++) {
+  for (size_t b = 0; b < x.shape(0) / batchsize; b++) {
     int start = b * batchsize, end = start + batchsize;
     const Tensor bx = CopyRows(x, start, end);
     const Tensor by = CopyRows(y, start, end);
-    const Tensor fea = Forward(kEval, bx);
     const auto ret = EvaluateOnBatch(bx, by);
     CopyDataToFrom(&loss, ret.first, batchsize, start, 0);
     CopyDataToFrom(&metric, ret.second, batchsize, start, 0);
@@ -230,18 +227,19 @@ std::pair<Tensor, Tensor> Evaluate(Tensor x, Tensor y, int batchsize) {
 
 std::pair<Tensor, Tensor> FeedForwardNet::EvaluateOnBatch(const Tensor& x,
                                                           const Tensor& y) {
-  const Tensor fea = Forward(kEval, bx);
-  const Tensor m = metric_->Forward(fea, by);
-  const Tensor l = loss_->Forward(fea, by);
+  int flag = kEval;
+  const Tensor fea = Forward(flag, x);
+  const Tensor m = metric_->Forward(fea, y);
+  const Tensor l = loss_->Forward(fea, y);
   return std::make_pair(m, l);
 }
 
-const Tensor FeedForwardNet::Predict(const Tensor& x, int batchsize) {
+const Tensor FeedForwardNet::Predict(const Tensor& x, size_t batchsize) {
   CHECK_GE(x.shape(0), batchsize);
   int num_extra_samples = x.shape(0) % batchsize;
-  const auto outshape = layers_.back().GetOutputSampleShape();
+  const auto outshape = layers_.back()->GetOutputSampleShape();
   Tensor y(Shape{x.shape(0), Product(outshape)}, x.device());
-  for (int b = 0; b < x.shape(0) / batchsize; b++) {
+  for (size_t b = 0; b < x.shape(0) / batchsize; b++) {
     int start = b * batchsize, end = start + batchsize;
     const Tensor bx = CopyRows(x, start, end);
     CopyDataToFrom(&y, PredictOnBatch(bx), batchsize * y.shape(1),
@@ -258,6 +256,6 @@ const Tensor FeedForwardNet::Predict(const Tensor& x, int batchsize) {
 }
 
 const Tensor FeedForwardNet::PredictOnBatch(const Tensor& x) {
-  return Foward(kEval, x);
+  return Forward(kEval, x);
 }
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/convolution.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/convolution.cc b/src/model/layer/convolution.cc
index c27960d..e4991a4 100644
--- a/src/model/layer/convolution.cc
+++ b/src/model/layer/convolution.cc
@@ -112,5 +112,9 @@ const std::pair<Tensor, vector<Tensor>> Convolution::Backward(
 
   return std::make_pair(input_grad, param_grad);
 }
-
+void Convolution::ToDevice(std::shared_ptr<Device> device) {
+  Layer::ToDevice(device);
+  weight_.ToDevice(device);
+  bias_.ToDevice(device);
+}
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/convolution.h
----------------------------------------------------------------------
diff --git a/src/model/layer/convolution.h b/src/model/layer/convolution.h
index 7ea5712..0e0b160 100644
--- a/src/model/layer/convolution.h
+++ b/src/model/layer/convolution.h
@@ -44,6 +44,7 @@ class Convolution : public Layer {
   const std::pair<Tensor, vector<Tensor>> Backward(int flag,
                                                    const Tensor &grad) override;
 
+  void ToDevice(std::shared_ptr<Device> device) override;
   size_t kernel_w() const { return kernel_w_; }
   size_t kernel_h() const { return kernel_h_; }
   size_t pad_w() const { return pad_w_; }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/cudnn_activation.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/cudnn_activation.cc b/src/model/layer/cudnn_activation.cc
index 4216fea..df728ce 100644
--- a/src/model/layer/cudnn_activation.cc
+++ b/src/model/layer/cudnn_activation.cc
@@ -54,6 +54,7 @@ void CudnnActivation::InitCudnn(size_t size, DataType dtype) {
 }
 
 const Tensor CudnnActivation::Forward(int flag, const Tensor& input) {
+  CHECK(buf_.empty());
   auto size = input.Size();
   DataType dtype = input.data_type();
   if (!has_init_cudnn_) {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/cudnn_convolution.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/cudnn_convolution.cc b/src/model/layer/cudnn_convolution.cc
index d5ac2a3..eb507b2 100644
--- a/src/model/layer/cudnn_convolution.cc
+++ b/src/model/layer/cudnn_convolution.cc
@@ -156,6 +156,7 @@ void CudnnConvolution::InitCudnn(const Tensor &input) {
 }
 
 const Tensor CudnnConvolution::Forward(int flag, const Tensor &input) {
+  CHECK(buf_.empty());
   CHECK_EQ(input.device()->lang(), kCuda);
   CHECK_EQ(input.nDim(), 4u);
   if (flag & kTrain) buf_.push(input);  // buffer the input for backward

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/cudnn_dropout.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/cudnn_dropout.cc b/src/model/layer/cudnn_dropout.cc
index 2e2e12b..f9b9dbf 100644
--- a/src/model/layer/cudnn_dropout.cc
+++ b/src/model/layer/cudnn_dropout.cc
@@ -106,6 +106,10 @@ const std::pair<Tensor, vector<Tensor>> CudnnDropout::Backward(
   }
   return std::make_pair(dx, param_grad);
 }
+void CudnnDropout::ToDevice(std::shared_ptr<Device> device) {
+  Dropout::ToDevice(device);
+  state.ToDevice(device);
+}
 }  // namespace singa
 #endif  // CUDNN_VERSION_MAJOR>=5
 #endif  // USE_CUDNN

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/cudnn_dropout.h
----------------------------------------------------------------------
diff --git a/src/model/layer/cudnn_dropout.h b/src/model/layer/cudnn_dropout.h
index 6809653..9e0cb9e 100644
--- a/src/model/layer/cudnn_dropout.h
+++ b/src/model/layer/cudnn_dropout.h
@@ -42,6 +42,7 @@ class CudnnDropout : public Dropout {
   const std::pair<Tensor, vector<Tensor>> Backward(int flag,
                                                    const Tensor& grad) override;
 
+  void ToDevice(std::shared_ptr<Device> device) override;
  private:
   /// Init cudnn related data structures.
   void InitCudnn(int size, DataType dtype, std::shared_ptr<Device> dev,

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/cudnn_pooling.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/cudnn_pooling.cc b/src/model/layer/cudnn_pooling.cc
index 6d7a5b1..e49a1ec 100644
--- a/src/model/layer/cudnn_pooling.cc
+++ b/src/model/layer/cudnn_pooling.cc
@@ -78,6 +78,7 @@ void CudnnPooling::InitCudnn(const Tensor &input) {
 }
 
 const Tensor CudnnPooling::Forward(int flag, const Tensor &input) {
+  CHECK(buf_.empty());
   CHECK_EQ(input.device()->lang(), kCuda);
   CHECK_EQ(input.nDim(), 4u);
   size_t batchsize = input.shape(0);

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/cudnn_softmax.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/cudnn_softmax.cc b/src/model/layer/cudnn_softmax.cc
index 77eab98..1d9e0b8 100644
--- a/src/model/layer/cudnn_softmax.cc
+++ b/src/model/layer/cudnn_softmax.cc
@@ -57,6 +57,7 @@ void CudnnSoftmax::InitCudnn(Shape shape, DataType dtype) {
 }
 
 const Tensor CudnnSoftmax::Forward(int flag, const Tensor& input) {
+  CHECK(buf_.empty());
   auto shape = input.shape();
   DataType dtype = input.data_type();
   if (!has_init_cudnn_) {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/layer/dense.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/dense.cc b/src/model/layer/dense.cc
index bad26a8..c6a9f8a 100644
--- a/src/model/layer/dense.cc
+++ b/src/model/layer/dense.cc
@@ -45,7 +45,9 @@ void Dense::Setup(const Shape& in_sample, const LayerConf &conf) {
 
 /// \copydoc Layer::Forward(int flag, const Tensor&)
 const Tensor Dense::Forward(int flag, const Tensor &input) {
+  CHECK(buf_.empty());
   Tensor output;
+  CHECK_EQ(input.nDim(), 2);
   if (transpose_)  // use the transposed version of weight_ for computing
     output = Mult(input, weight_);
   else
@@ -81,6 +83,7 @@ const std::pair<Tensor, vector<Tensor>> Dense::Backward(int flag,
 }
 
 void Dense::ToDevice(std::shared_ptr<Device> device) {
+  Layer::ToDevice(device);
   weight_.ToDevice(device);
   bias_.ToDevice(device);
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/metric/accuracy.cc
----------------------------------------------------------------------
diff --git a/src/model/metric/accuracy.cc b/src/model/metric/accuracy.cc
new file mode 100644
index 0000000..1b667b1
--- /dev/null
+++ b/src/model/metric/accuracy.cc
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/model/metric.h"
+#include <algorithm>
+namespace singa {
+
+Tensor Accuracy::Match(const Tensor& predict, const vector<int>& target) {
+  Tensor prediction(predict.shape());
+  prediction.CopyData(predict);
+  size_t batchsize = target.size();
+  size_t nb_classes = prediction.Size() / batchsize;
+  // each row of prediction is the prob distribution for one sample
+  CHECK_EQ(prediction.shape().at(0), batchsize);
+  // TODO(wangwei) CloneToDevice(host);
+  const float* prob = prediction.data<float>();
+  float* score = new float[batchsize];
+  for (size_t b = 0; b < batchsize; b++) {
+    vector<std::pair<float, int>> prob_class;
+    for (size_t c = 0; c < nb_classes; c++) {
+      prob_class.push_back(std::make_pair(prob[b * nb_classes + c], c));
+    }
+    std::partial_sort(prob_class.begin(), prob_class.begin() + top_k_,
+                      prob_class.end(), std::greater<std::pair<float, int>>());
+
+    for (size_t k = 0; k < top_k_; k++)
+      if (prob_class.at(k).second == target.at(b)) score[b] = 1;
+  }
+  Tensor ret(Shape{batchsize});
+  ret.CopyDataFromHostPtr(score, batchsize);
+  return ret;
+}
+
+// TODO(wangwei) consider multi-label cases, where target is of shape
+// nb_samples * nb_classes
+Tensor Accuracy::Forward(const Tensor& prediction, const Tensor& t) {
+  Tensor target(t.shape(), t.data_type());
+  target.CopyData(t);
+  vector<int> target_vec;
+  // TODO(wangwei) copy target to host.
+  const int* target_value = target.data<int>();
+  for (size_t i = 0; i < target.Size(); i++)
+    target_vec.push_back(target_value[i]);
+  return Match(prediction, target_vec);
+}
+
+}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/src/model/metric/accuracy.h
----------------------------------------------------------------------
diff --git a/src/model/metric/accuracy.h b/src/model/metric/accuracy.h
deleted file mode 100644
index 69bd96b..0000000
--- a/src/model/metric/accuracy.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef SINGA_MODEL_METRIC_ACCURACY_H_
-#define SINGA_MODEL_METRIC_ACCURACY_H_
-#include "singa/model/metric.h"
-#include <algorithm>
-namespace singa {
-
-/// Compute the accuray of the prediction, which is matched against the
-/// ground truth labels.
-/// TODO(wangwei) consider multi-label cases.
-class Accuracy : public Metric<Tensor> {
- public:
-  /// Set meta fields from user configurations.
-  void Setup(const MetricConf& conf) override { top_k_ = conf.top_k(); }
-
-  /// Check the prediction against the target (ground truth) for each data
-  /// sample. The returned Tensor has a float value for each sample, 0 for wrong
-  /// and 1 for correct. Users can call Sum(const Tensor&) / Tensor::Size() to
-  /// get the accuracy.
-  Tensor Forward(const Tensor& prediction, const Tensor& target);
-
- private:
-  /// \copydoc Match(const Tensor&, const Tensor&);
-  Tensor Match(const Tensor& prediction, const vector<int>& target);
-  /// If the ground truth label is in the top k predicted labels, then the
-  /// prediction is correct.
-  size_t top_k_ = 1;
-};
-
-Tensor Accuracy::Match(const Tensor& prediction, const vector<int>& target) {
-  size_t batchsize = target.size();
-  size_t nb_classes = prediction.Size() / batchsize;
-  // each row of prediction is the prob distribution for one sample
-  CHECK_EQ(prediction.shape().at(0), batchsize);
-  // TODO(wangwei) CloneToDevice(host);
-  const float* prob = prediction.data<float>();
-  float* score = new float[batchsize];
-  for (size_t b = 0; b < batchsize; b++) {
-    vector<std::pair<float, int>> prob_class;
-    for (size_t c = 0; c < nb_classes; c++) {
-      prob_class.push_back(std::make_pair(prob[b * nb_classes + c], c));
-    }
-    std::partial_sort(prob_class.begin(), prob_class.begin() + top_k_,
-                      prob_class.end(), std::greater<std::pair<float, int>>());
-
-    for (size_t k = 0; k < top_k_; k++)
-      if (prob_class.at(k).second == target.at(b)) score[b] = 1;
-  }
-  Tensor ret(Shape{batchsize});
-  ret.CopyDataFromHostPtr(score, batchsize);
-  return ret;
-}
-
-// TODO(wangwei) consider multi-label cases, where target is of shape
-// nb_samples * nb_classes
-Tensor Accuracy::Forward(const Tensor& prediction, const Tensor& target) {
-  vector<int> target_vec;
-  // TODO(wangwei) copy target to host.
-  const int* target_value = target.data<int>();
-  for (size_t i = 0; i < target.Size(); i++)
-    target_vec.push_back(target_value[i]);
-  return Match(prediction, target_vec);
-}
-
-}  // namespace singa
-
-#endif  // SINGA_MODEL_METRIC_ACCURACY_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/cf1d8418/test/singa/test_accuracy.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_accuracy.cc b/test/singa/test_accuracy.cc
index dc7719b..4ff14c0 100644
--- a/test/singa/test_accuracy.cc
+++ b/test/singa/test_accuracy.cc
@@ -20,7 +20,7 @@
 *************************************************************/
 
 #include "gtest/gtest.h"
-#include "../src/model/metric/accuracy.h"
+#include "singa/model/metric.h"
 
 TEST(Accuracy, Compute) {
   singa::Accuracy acc;


Mime
View raw message