singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [1/2] incubator-singa git commit: SINGA-91 - Add SoftmaxLayer and ArgSortLayer
Date Sun, 18 Oct 2015 14:47:14 GMT
Repository: incubator-singa
Updated Branches:
  refs/heads/master 497dafc71 -> 0a228c8cc


SINGA-91 - Add SoftmaxLayer and ArgSortLayer

SoftmaxLayer applies the Softmax function against its source layer to compute its probability
distribution over all labels.

ArgSortLayer sorts labels based on their scores (e.g., probability) in descending order.
Configuration for ArgSortLayer is like argsort_conf{ topk: 1}. Topk results will be extracted.
Connecting ArgSortLayer to a CSVOutputLayer, we can dump the topk labels
of each instance into one line.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/ef9a1113
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/ef9a1113
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/ef9a1113

Branch: refs/heads/master
Commit: ef9a11136538713e1f707ebb87a0ae07bde764d2
Parents: 497dafc
Author: Wei Wang <wangwei@comp.nus.edu.sg>
Authored: Thu Oct 15 22:30:32 2015 +0800
Committer: Wei Wang <wangwei@comp.nus.edu.sg>
Committed: Thu Oct 15 22:34:00 2015 +0800

----------------------------------------------------------------------
 Makefile.am                                    |  2 +
 examples/cifar10/job.conf                      | 17 +++++-
 include/singa/neuralnet/neuron_layer/argsort.h | 52 +++++++++++++++++
 include/singa/neuralnet/neuron_layer/softmax.h | 40 +++++++++++++
 include/singa/neuralnet/output_layer/csv.h     |  2 +-
 include/singa/neuralnet/output_layer/record.h  |  2 +-
 src/driver.cc                                  |  6 ++
 src/neuralnet/loss_layer/softmax.cc            |  1 +
 src/neuralnet/neuron_layer/argsort.cc          | 57 +++++++++++++++++++
 src/neuralnet/neuron_layer/softmax.cc          | 62 +++++++++++++++++++++
 src/proto/job.proto                            | 11 +++-
 11 files changed, 245 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index b3e0418..6e02c6a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -45,6 +45,7 @@ SINGA_SRCS := src/driver.cc \
               src/neuralnet/output_layer/record.cc \
               src/neuralnet/loss_layer/euclidean.cc \
               src/neuralnet/loss_layer/softmax.cc \
+              src/neuralnet/neuron_layer/argsort.cc \
               src/neuralnet/neuron_layer/convolution.cc \
               src/neuralnet/neuron_layer/dropout.cc \
               src/neuralnet/neuron_layer/inner_product.cc \
@@ -54,6 +55,7 @@ SINGA_SRCS := src/driver.cc \
               src/neuralnet/neuron_layer/relu.cc \
               src/neuralnet/neuron_layer/sigmoid.cc \
               src/neuralnet/neuron_layer/stanh.cc \
+              src/neuralnet/neuron_layer/softmax.cc \
               src/neuralnet/neuralnet.cc \
               src/comm/socket.cc \
               src/comm/msg.cc \

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index 9ecb0c3..a1fefd8 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -1,6 +1,6 @@
 name: "cifar10-convnet"
 train_steps: 1000
-test_steps: 10
+test_steps: 100
 test_freq: 300
 disp_freq: 30
 #checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
@@ -221,6 +221,17 @@ neuralnet {
       }
     }
   }
+#  layer {
+#   name : "softmax"
+#   type: kSoftmax
+#   srclayers: "ip1"
+#  }
+#
+#  layer {
+#   name : "argsort"
+#   type: kArgSort
+#   srclayers: "softmax"
+#  }
   layer{
     name: "loss"
     type: kSoftmaxLoss
@@ -230,11 +241,11 @@ neuralnet {
     srclayers:"ip1"
     srclayers: "data"
   }
-# uncomment "output" layer and comment "loss" layer to extract features from ip1
+# uncomment "output" layer and comment "loss" layer to extract features from argsort
 #  layer {
 #    name : "output"
 #    type: kCSVOutput
-#    srclayers: "ip1"
+#    srclayers: "argsort"
 #    store_conf {
 #      path: "examples/cifar10/out.csv"
 #    }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/include/singa/neuralnet/neuron_layer/argsort.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/argsort.h b/include/singa/neuralnet/neuron_layer/argsort.h
new file mode 100644
index 0000000..1135aa1
--- /dev/null
+++ b/include/singa/neuralnet/neuron_layer/argsort.h
@@ -0,0 +1,52 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_NEURON_LAYER_ARGSORT_H_
+#define SINGA_NEURALNET_NEURON_LAYER_ARGSORT_H_
+
+#include <glog/logging.h>
+#include <vector>
+#include "singa/neuralnet/layer.h"
+#include "singa/proto/job.pb.h"
+namespace singa {
+
+/**
+ * ArgSort layer used to get topk prediction labels.
+ *
+ * It sort the labels based on its score (e.g., probability) from large to
+ * small. Topk labels will be kepted in the data field. It should not be called
+ * during training because this layer does not implement ComputeGradient()
+ * function.
+ */
+class ArgSortLayer : public NeuronLayer {
+ public:
+  void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+  void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+  void ComputeGradient(int flag, const vector<Layer*>& srclayers) {
+    LOG(FATAL) << "Not Implemented";
+  }
+
+ private:
+  int batchsize_, dim_;
+  int topk_;
+};
+} /* singa */
+#endif  // SINGA_NEURALNET_NEURON_LAYER_ARGSORT_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/include/singa/neuralnet/neuron_layer/softmax.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer/softmax.h b/include/singa/neuralnet/neuron_layer/softmax.h
new file mode 100644
index 0000000..6132411
--- /dev/null
+++ b/include/singa/neuralnet/neuron_layer/softmax.h
@@ -0,0 +1,40 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_NEURALNET_NEURON_LAYER_SOFTMAX_H_
+#define SINGA_NEURALNET_NEURON_LAYER_SOFTMAX_H_
+
+#include <vector>
+#include "singa/neuralnet/layer.h"
+#include "singa/proto/job.pb.h"
+namespace singa {
+
+/**
+ * Softmax layer.
+ */
+class SoftmaxLayer : public NeuronLayer {
+ public:
+  void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+  void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+  void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+};
+} /* singa */
+#endif  // SINGA_NEURALNET_NEURON_LAYER_SOFTMAX_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/include/singa/neuralnet/output_layer/csv.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/output_layer/csv.h b/include/singa/neuralnet/output_layer/csv.h
index cb1545a..439db8c 100644
--- a/include/singa/neuralnet/output_layer/csv.h
+++ b/include/singa/neuralnet/output_layer/csv.h
@@ -37,8 +37,8 @@ class CSVOutputLayer : public OutputLayer {
   void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
 
  private:
-  io::Store* store_ = nullptr;
   int inst_ = 0;
+  io::Store* store_ = nullptr;
 };
 }  // namespace singa
 #endif  // SINGA_NEURALNET_OUTPUT_LAYER_CSV_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/include/singa/neuralnet/output_layer/record.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/output_layer/record.h b/include/singa/neuralnet/output_layer/record.h
index a2adb0f..cd652d5 100644
--- a/include/singa/neuralnet/output_layer/record.h
+++ b/include/singa/neuralnet/output_layer/record.h
@@ -35,8 +35,8 @@ class RecordOutputLayer : public OutputLayer {
   void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
 
  private:
-  io::Store* store_ = nullptr;
   int inst_ = 0;  //!< instance No.
+  io::Store* store_ = nullptr;
 };
 }  // namespace singa
 #endif  // SINGA_NEURALNET_OUTPUT_LAYER_RECORD_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 35d1f38..ba65d3e 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -45,6 +45,7 @@
 #include "singa/neuralnet/input_layer/store.h"
 #include "singa/neuralnet/loss_layer/euclidean.h"
 #include "singa/neuralnet/loss_layer/softmax.h"
+#include "singa/neuralnet/neuron_layer/argsort.h"
 #include "singa/neuralnet/neuron_layer/convolution.h"
 #include "singa/neuralnet/neuron_layer/dropout.h"
 #include "singa/neuralnet/neuron_layer/inner_product.h"
@@ -54,6 +55,7 @@
 #include "singa/neuralnet/neuron_layer/relu.h"
 #include "singa/neuralnet/neuron_layer/sigmoid.h"
 #include "singa/neuralnet/neuron_layer/stanh.h"
+#include "singa/neuralnet/neuron_layer/softmax.h"
 #include "singa/neuralnet/output_layer/record.h"
 #include "singa/neuralnet/output_layer/csv.h"
 
@@ -88,6 +90,8 @@ void Driver::Init(int argc, char **argv) {
 
   RegisterLayer<BridgeDstLayer, int>(kBridgeDst);
   RegisterLayer<BridgeSrcLayer, int>(kBridgeSrc);
+
+  RegisterLayer<ArgSortLayer, int>(kArgSort);
   RegisterLayer<ConvolutionLayer, int>(kConvolution);
   RegisterLayer<CConvolutionLayer, int>(kCConvolution);
   RegisterLayer<CPoolingLayer, int>(kCPooling);
@@ -110,6 +114,8 @@ void Driver::Init(int argc, char **argv) {
   RegisterLayer<SoftmaxLossLayer, int>(kSoftmaxLoss);
   RegisterLayer<SplitLayer, int>(kSplit);
   RegisterLayer<STanhLayer, int>(kSTanh);
+  RegisterLayer<SoftmaxLayer, int>(kSoftmax);
+
 #ifdef USE_LMDB
   RegisterLayer<LMDBDataLayer, int>(kLMDBData);
 #endif

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/src/neuralnet/loss_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc
index 058462c..497c287 100644
--- a/src/neuralnet/loss_layer/softmax.cc
+++ b/src/neuralnet/loss_layer/softmax.cc
@@ -47,6 +47,7 @@ void SoftmaxLossLayer::Setup(const LayerProto& proto,
   topk_ = proto.softmaxloss_conf().topk();
   scale_ = proto.softmaxloss_conf().scale();
 }
+
 void SoftmaxLossLayer::ComputeFeature(int flag,
     const vector<Layer*>& srclayers) {
   Shape<2> s = Shape2(batchsize_, dim_);

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/src/neuralnet/neuron_layer/argsort.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/argsort.cc b/src/neuralnet/neuron_layer/argsort.cc
new file mode 100644
index 0000000..a515485
--- /dev/null
+++ b/src/neuralnet/neuron_layer/argsort.cc
@@ -0,0 +1,57 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+
+#include "singa/neuralnet/neuron_layer/argsort.h"
+#include <algorithm>
+
+namespace singa {
+
+void ArgSortLayer::Setup(const LayerProto& proto,
+    const vector<Layer*>& srclayers) {
+  CHECK_EQ(srclayers.size(), 1);
+  NeuronLayer::Setup(proto, srclayers);
+  batchsize_ = srclayers[0]->data(this).shape()[0];
+  dim_ = srclayers[0]->data(this).count() / batchsize_;
+  topk_ = proto.argsort_conf().topk();
+  data_.Reshape(vector<int>{batchsize_, topk_});
+}
+
+void ArgSortLayer::ComputeFeature(int flag,
+    const vector<Layer*>& srclayers) {
+  // TODO(wangwei) check flag to ensure it is not called in training phase
+  const float* srcptr = srclayers.at(0)->data(this).cpu_data();
+  float* ptr = data_.mutable_cpu_data();
+  for (int n = 0; n < batchsize_; n++) {
+    vector<std::pair<float, int> > vec;
+    for (int j = 0; j < dim_; ++j)
+      vec.push_back(std::make_pair(srcptr[j], j));
+    std::partial_sort(vec.begin(), vec.begin() + topk_, vec.end(),
+                      std::greater<std::pair<float, int> >());
+
+    for (int j = 0; j < topk_; ++j)
+      ptr[j] = static_cast<float> (vec.at(j).second);
+    ptr += topk_;
+    srcptr += dim_;
+  }
+}
+
+}  /* singa */

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/src/neuralnet/neuron_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/softmax.cc b/src/neuralnet/neuron_layer/softmax.cc
new file mode 100644
index 0000000..be2f337
--- /dev/null
+++ b/src/neuralnet/neuron_layer/softmax.cc
@@ -0,0 +1,62 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/neuralnet/neuron_layer/softmax.h"
+
+namespace singa {
+
+using namespace mshadow;
+using mshadow::cpu;
+
+using mshadow::Shape;
+using mshadow::Shape1;
+using mshadow::Shape2;
+using mshadow::Tensor;
+
+using std::vector;
+
+void SoftmaxLayer::Setup(const LayerProto& proto,
+    const vector<Layer*>& srclayers) {
+  CHECK_EQ(srclayers.size(), 1);
+  NeuronLayer::Setup(proto, srclayers);
+  data_.Reshape(srclayers[0]->data(this).shape());
+}
+
+void SoftmaxLayer::ComputeFeature(int flag,
+    const vector<Layer*>& srclayers) {
+  int batchsize = data_.shape()[0];
+  int dim = data_.count() / batchsize;
+  Shape<2> s = Shape2(batchsize, dim);
+  Tensor<cpu, 2> prob(data_.mutable_cpu_data(), s);
+  Tensor<cpu, 2> src(srclayers[0]->mutable_data(this)->mutable_cpu_data(), s);
+  Softmax(prob, src);
+}
+
+void SoftmaxLayer::ComputeGradient(int flag,
+    const vector<Layer*>& srclayers) {
+  int batchsize = data_.shape()[0];
+  for (int n = 0; n < batchsize; n++) {
+    // TODO(wangwei) finish the code using new math API
+    // gxi=[(gyi+gyi*yi)-\sum_k(gyk*yk)]*yi
+  }
+}
+
+}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ef9a1113/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 2ccd674..eb1b876 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -184,6 +184,8 @@ message LayerProto {
   optional string user_type =21;
 
   // proto for the specific layer
+  // configuration for argsort layer
+  optional ArgSortProto argsort_conf = 52;
   // configuration for convolution layer
   optional ConvolutionProto convolution_conf = 30;
   // configuration for concatenation layer
@@ -219,8 +221,6 @@ message LayerProto {
   // configuration for store input layers
   optional StoreProto store_conf = 51;
 
-
-
   // overrides the partition dimension for neural net
   optional int32 partition_dim = 60 [default = -1];
   // names of parameters shared from other layers
@@ -338,6 +338,11 @@ message SoftmaxLossProto {
   optional float scale = 30 [default = 1];
 }
 
+message ArgSortProto {
+  // keep labels with topk scores
+  optional int32 topk = 1 [default = 1];
+}
+
 message ConvolutionProto {
   // The number of outputs for the layer
   required int32 num_filters = 1;
@@ -552,6 +557,7 @@ enum LayerType {
   kRGBImage = 10;
   // Neuron layers
   //  - Feature transformation
+  kArgSort = 35;
   kConvolution = 1;
   kCConvolution = 27;
   kCPooling = 28;
@@ -564,6 +570,7 @@ enum LayerType {
   kRBMHid = 24;
   kSigmoid = 26;
   kSTanh = 14;
+  kSoftmax = 34;
   // Loss layers
   //  - Compute objective loss
   kSoftmaxLoss = 11;


Mime
View raw message