singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [2/2] incubator-singa git commit: SINGA-213 Implement Encoder and Decoder for CSV
Date Mon, 11 Jul 2016 06:59:25 GMT
 SINGA-213 Implement Encoder and Decoder for CSV

 Rename text_encoder and text_decoder to csv_encoder and csv_decoder.
 Rename jpg2proto and proto2jpg to JPGEncoder and JPGDecoder.
 Rename the corresponding files.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/49319223
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/49319223
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/49319223

Branch: refs/heads/dev
Commit: 49319223316bbaffc9082f0cb42e931111dbf6f1
Parents: 7444f0a
Author: Wei Wang <wangwei@comp.nus.edu.sg>
Authored: Mon Jul 11 14:57:19 2016 +0800
Committer: Wei Wang <wangwei@comp.nus.edu.sg>
Committed: Mon Jul 11 14:57:19 2016 +0800

----------------------------------------------------------------------
 include/singa/io/decoder.h          | 10 ++--
 include/singa/io/encoder.h          |  8 +--
 src/io/csv_decoder.cc               | 55 +++++++++++++++++++++
 src/io/csv_encoder.cc               | 43 ++++++++++++++++
 src/io/jpg2proto_encoder.cc         | 83 -------------------------------
 src/io/jpg_decoder.cc               | 75 ++++++++++++++++++++++++++++
 src/io/jpg_encoder.cc               | 83 +++++++++++++++++++++++++++++++
 src/io/proto2jpg_decoder.cc         | 75 ----------------------------
 src/io/textfile_decoder.cc          | 54 --------------------
 src/io/textfile_encoder.cc          | 43 ----------------
 test/singa/test_csv.cc              | 60 +++++++++++++++++++++++
 test/singa/test_decoder.cc          | 84 --------------------------------
 test/singa/test_jpg.cc              | 84 ++++++++++++++++++++++++++++++++
 test/singa/test_textfile_decoder.cc | 60 -----------------------
 14 files changed, 409 insertions(+), 408 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/include/singa/io/decoder.h
----------------------------------------------------------------------
diff --git a/include/singa/io/decoder.h b/include/singa/io/decoder.h
index 7d47d6a..bf9a1bc 100644
--- a/include/singa/io/decoder.h
+++ b/include/singa/io/decoder.h
@@ -55,9 +55,9 @@ class JPGDecoder : public Decoder {
 };
 #endif
 
-/// Decode the string and convert it into a text
-/// tensor (dtype is kFloat32) and a label tensor (dtype is kInt).
-class TextDecoder : public Decoder {
+/// Decode the string of csv formated data  into data tensor
+/// (dtype is kFloat32) and optionally a label tensor (dtype is kInt).
+class CSVDecoder : public Decoder {
  public:
   void Setup(const DecoderConf& conf) override {
     has_label_ = conf.has_label();
@@ -67,8 +67,8 @@ class TextDecoder : public Decoder {
   const bool has_label() const { return has_label_; }
 
  private:
-  /// Indicate the dimension order for the output image tensor.
-  bool has_label_ = true;
+  /// if ture the first value is the label
+  bool has_label_ = false;
 };
 } // namespace singa
 #endif // SINGA_IO_DECODER_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/include/singa/io/encoder.h
----------------------------------------------------------------------
diff --git a/include/singa/io/encoder.h b/include/singa/io/encoder.h
index 90cea0e..5ca7b21 100644
--- a/include/singa/io/encoder.h
+++ b/include/singa/io/encoder.h
@@ -60,12 +60,12 @@ class JPGEncoder : public Encoder {
 };
 #endif  // USE_OPENCV
 
-/// Convert a set of tensors parsed from csv file into strings
-class TextEncoder : public Encoder {
+/// Convert values from tensors into a csv formated string.
+class CSVEncoder : public Encoder {
  public:
   void Setup(const EncoderConf& conf) override {}
-  /// 'data' has two tesors, one for the text vector (1D) and one for the
-  /// label. The text tensor's data type is kFloat.
+  /// 'data' has two tesors, one for the data vector (1D) and one for the
+  /// label. The data tensor's data type is kFloat.
   /// The label tensor's data type is kInt.
   std::string Encode(vector<Tensor>& data) override;
 };

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/csv_decoder.cc
----------------------------------------------------------------------
diff --git a/src/io/csv_decoder.cc b/src/io/csv_decoder.cc
new file mode 100644
index 0000000..0c11028
--- /dev/null
+++ b/src/io/csv_decoder.cc
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/io/decoder.h"
+#include <string>
+#include <sstream>
+
+const int kMaxCSVBufSize = 40960;
+
+namespace singa {
+
+std::vector<Tensor> CSVDecoder::Decode(std::string value) {
+  std::vector<Tensor> output;
+  std::stringstream ss;
+  ss.str(value);
+  int l = 0;
+  if (has_label_ == true)
+    ss >> l;
+  std::string str;
+  float d[kMaxCSVBufSize];
+  size_t size = 0;
+  while (std::getline(ss, str, ',')) {
+    float temp;
+    if (std::stringstream(str) >> temp) {
+      CHECK_LE(size, kMaxCSVBufSize - 1);
+      d[size++] = temp;
+    }
+  }
+
+  Tensor data(Shape {size}, kFloat32);
+  data.CopyDataFromHostPtr(d, size);
+  output.push_back(data);
+  if (has_label_ == true) {
+    Tensor label(Shape {1}, kInt);
+    label.CopyDataFromHostPtr(&l, 1);
+    output.push_back(label);
+  }
+  return output;
+}
+}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/csv_encoder.cc
----------------------------------------------------------------------
diff --git a/src/io/csv_encoder.cc b/src/io/csv_encoder.cc
new file mode 100644
index 0000000..1b797a9
--- /dev/null
+++ b/src/io/csv_encoder.cc
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/io/encoder.h"
+#include <sstream>
+
+namespace singa {
+
+std::string CSVEncoder::Encode(vector<Tensor>& data) {
+  CHECK_GE(data.size(), 1);
+  size_t size = data[0].Size();
+  const float* value = data[0].data<float>();
+  std::string des = "";
+  if (data.size() == 2) {
+    const float label = (const float)data[1].data<int>()[0];
+    std::ostringstream buff;
+    buff << label;
+    des += buff.str() + ',';
+  }
+  for (size_t i = 0; i < size; i++) {
+    std::ostringstream buff;
+    buff << value[i];
+    if (i == size - 1) des += buff.str();
+    else des += buff.str() + ',';
+  }
+  return des;
+}
+}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/jpg2proto_encoder.cc
----------------------------------------------------------------------
diff --git a/src/io/jpg2proto_encoder.cc b/src/io/jpg2proto_encoder.cc
deleted file mode 100644
index c5d81b8..0000000
--- a/src/io/jpg2proto_encoder.cc
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "singa/io/encoder.h"
-
-#ifdef USE_OPENCV
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
-
-namespace singa {
-
-std::string JPG2ProtoEncoder::Encode(vector<Tensor>& data) {
-  // suppose image: image, data[1]: label
-  CHECK_LE(data.size(), 2u);
-  const Tensor& image = data.at(0);
-  CHECK_EQ(image.nDim(), 3u);
-  CHECK_EQ(image.data_type(), kUChar) << "Data type " << image.data_type()
-    << " is invalid for an raw image";
-  const auto* raw = image.data<unsigned char>();
-  cv::Mat mat;
-  if (image_dim_order_ == "HWC") {
-    size_t height = image.shape(0), width = image.shape(1),
-           channel = image.shape(2);
-    mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0));
-    for (size_t i = 0; i < height; i++)
-      for (size_t j = 0; j < width; j++)
-        for (size_t k = 0; k < channel; k++)
-          mat.at<cv::Vec3b>(i, j)[k] =
-              raw[i * width * channel + j * channel + k];
-  } else if (image_dim_order_ == "CHW") {
-    size_t channel = image.shape(0), height = image.shape(1),
-           width = image.shape(2);
-    mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0));
-    for (size_t i = 0; i < height; i++)
-      for (size_t j = 0; j < width; j++)
-        for (size_t k = 0; k < channel; k++)
-          mat.at<cv::Vec3b>(i, j)[k] = raw[k * height * width + i * width + j];
-  } else {
-    LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_
-               << " Only support 'HWC' and 'CHW'";
-  }
-
-  // encode image with jpg format
-  std::vector<uchar> buff;
-  std::vector<int> param = std::vector<int>(2);
-  param[0] = CV_IMWRITE_JPEG_QUALITY;
-  param[1] = 100;  // default is 95
-  cv::imencode(".jpg", mat, buff, param);
-  std::string buf(buff.begin(), buff.end());
-
-  std::string output;
-  ImageRecord record;
-  for (size_t i = 0; i < image.nDim(); i++)
-    record.add_shape(image.shape(i));
-  record.set_pixel(buf);
-
-  // suppose each image is attached with at most one label
-  if (data.size() == 2) {
-    const int* label = data[1].data<int>();
-    CHECK_EQ(label[0], 2);
-    record.add_label(label[0]);
-  }
-
-  record.SerializeToString(&output);
-  return output;
-}
-}  // namespace singa
-#endif  // USE_OPENCV

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/jpg_decoder.cc
----------------------------------------------------------------------
diff --git a/src/io/jpg_decoder.cc b/src/io/jpg_decoder.cc
new file mode 100644
index 0000000..f885537
--- /dev/null
+++ b/src/io/jpg_decoder.cc
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/io/decoder.h"
+
+#ifdef USE_OPENCV
+
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+namespace singa {
+
+std::vector<Tensor> JPGDecoder::Decode(std::string value) {
+  std::vector<Tensor> output;
+
+  ImageRecord record;
+  record.ParseFromString(value);
+  std::vector<uchar> pixel(record.pixel().begin(), record.pixel().end());
+
+  // decode image
+  cv::Mat mat = cv::imdecode(cv::Mat(pixel), CV_LOAD_IMAGE_COLOR);
+  size_t height = mat.rows, width = mat.cols, channel = mat.channels();
+  Shape shape(record.shape().begin(), record.shape().end());
+  CHECK_EQ(shape[0], height);
+  CHECK_EQ(shape[1], width);
+  CHECK_EQ(shape[2], channel);
+  Tensor image(shape);
+
+  float* data = new float[image.Size()];
+  if (image_dim_order_ == "CHW") {
+    for (size_t i = 0; i < height; i++)
+      for (size_t j = 0; j < width; j++)
+        for (size_t k = 0; k < channel; k++)
+          data[k * height * width + i * width + j] = static_cast<float>(
+              static_cast<int>(mat.at<cv::Vec3b>(i, j)[k]));
+  } else if (image_dim_order_ == "HWC") {
+
+    for (size_t i = 0; i < height; i++)
+      for (size_t j = 0; j < width; j++)
+        for (size_t k = 0; k < channel; k++)
+          data[i * width * channel + j * channel + k] =
+              static_cast<float>(static_cast<int>(mat.at<cv::Vec3b>(i,
j)[k]));
+  } else {
+    LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_
+               << " Only support 'HWC' and 'CHW'";
+  }
+  image.CopyDataFromHostPtr<float>(data, image.Size());
+  output.push_back(image);
+  delete data;
+
+  if (record.label_size()) {
+    Tensor label(Shape{1}, &defaultDevice, kInt);
+    int labelid = record.label(0);
+    label.CopyDataFromHostPtr(&labelid, 1);
+    output.push_back(label);
+  }
+  return output;
+}
+}  // namespace singa
+#endif

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/jpg_encoder.cc
----------------------------------------------------------------------
diff --git a/src/io/jpg_encoder.cc b/src/io/jpg_encoder.cc
new file mode 100644
index 0000000..9db799d
--- /dev/null
+++ b/src/io/jpg_encoder.cc
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/io/encoder.h"
+
+#ifdef USE_OPENCV
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+namespace singa {
+
+std::string JPGEncoder::Encode(vector<Tensor>& data) {
+  // suppose image: image, data[1]: label
+  CHECK_LE(data.size(), 2u);
+  const Tensor& image = data.at(0);
+  CHECK_EQ(image.nDim(), 3u);
+  CHECK_EQ(image.data_type(), kUChar) << "Data type " << image.data_type()
+    << " is invalid for an raw image";
+  const auto* raw = image.data<unsigned char>();
+  cv::Mat mat;
+  if (image_dim_order_ == "HWC") {
+    size_t height = image.shape(0), width = image.shape(1),
+           channel = image.shape(2);
+    mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0));
+    for (size_t i = 0; i < height; i++)
+      for (size_t j = 0; j < width; j++)
+        for (size_t k = 0; k < channel; k++)
+          mat.at<cv::Vec3b>(i, j)[k] =
+              raw[i * width * channel + j * channel + k];
+  } else if (image_dim_order_ == "CHW") {
+    size_t channel = image.shape(0), height = image.shape(1),
+           width = image.shape(2);
+    mat = cv::Mat(height, width, CV_8UC3, cv::Scalar(0, 0, 0));
+    for (size_t i = 0; i < height; i++)
+      for (size_t j = 0; j < width; j++)
+        for (size_t k = 0; k < channel; k++)
+          mat.at<cv::Vec3b>(i, j)[k] = raw[k * height * width + i * width + j];
+  } else {
+    LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_
+               << " Only support 'HWC' and 'CHW'";
+  }
+
+  // encode image with jpg format
+  std::vector<uchar> buff;
+  std::vector<int> param = std::vector<int>(2);
+  param[0] = CV_IMWRITE_JPEG_QUALITY;
+  param[1] = 100;  // default is 95
+  cv::imencode(".jpg", mat, buff, param);
+  std::string buf(buff.begin(), buff.end());
+
+  std::string output;
+  ImageRecord record;
+  for (size_t i = 0; i < image.nDim(); i++)
+    record.add_shape(image.shape(i));
+  record.set_pixel(buf);
+
+  // suppose each image is attached with at most one label
+  if (data.size() == 2) {
+    const int* label = data[1].data<int>();
+    CHECK_EQ(label[0], 2);
+    record.add_label(label[0]);
+  }
+
+  record.SerializeToString(&output);
+  return output;
+}
+}  // namespace singa
+#endif  // USE_OPENCV

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/proto2jpg_decoder.cc
----------------------------------------------------------------------
diff --git a/src/io/proto2jpg_decoder.cc b/src/io/proto2jpg_decoder.cc
deleted file mode 100644
index 0125ea3..0000000
--- a/src/io/proto2jpg_decoder.cc
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "singa/io/decoder.h"
-
-#ifdef USE_OPENCV
-
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
-
-namespace singa {
-
-std::vector<Tensor> Proto2JPGDecoder::Decode(std::string value) {
-  std::vector<Tensor> output;
-
-  ImageRecord record;
-  record.ParseFromString(value);
-  std::vector<uchar> pixel(record.pixel().begin(), record.pixel().end());
-
-  // decode image
-  cv::Mat mat = cv::imdecode(cv::Mat(pixel), CV_LOAD_IMAGE_COLOR);
-  size_t height = mat.rows, width = mat.cols, channel = mat.channels();
-  Shape shape(record.shape().begin(), record.shape().end());
-  CHECK_EQ(shape[0], height);
-  CHECK_EQ(shape[1], width);
-  CHECK_EQ(shape[2], channel);
-  Tensor image(shape);
-
-  float* data = new float[image.Size()];
-  if (image_dim_order_ == "CHW") {
-    for (size_t i = 0; i < height; i++)
-      for (size_t j = 0; j < width; j++)
-        for (size_t k = 0; k < channel; k++)
-          data[k * height * width + i * width + j] = static_cast<float>(
-              static_cast<int>(mat.at<cv::Vec3b>(i, j)[k]));
-  } else if (image_dim_order_ == "HWC") {
-
-    for (size_t i = 0; i < height; i++)
-      for (size_t j = 0; j < width; j++)
-        for (size_t k = 0; k < channel; k++)
-          data[i * width * channel + j * channel + k] =
-              static_cast<float>(static_cast<int>(mat.at<cv::Vec3b>(i,
j)[k]));
-  } else {
-    LOG(FATAL) << "Unknow dimension order for images " << image_dim_order_
-               << " Only support 'HWC' and 'CHW'";
-  }
-  image.CopyDataFromHostPtr<float>(data, image.Size());
-  output.push_back(image);
-  delete data;
-
-  if (record.label_size()) {
-    Tensor label(Shape{1}, &defaultDevice, kInt);
-    int labelid = record.label(0);
-    label.CopyDataFromHostPtr(&labelid, 1);
-    output.push_back(label);
-  }
-  return output;
-}
-}  // namespace singa
-#endif

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/textfile_decoder.cc
----------------------------------------------------------------------
diff --git a/src/io/textfile_decoder.cc b/src/io/textfile_decoder.cc
deleted file mode 100644
index 221fb9f..0000000
--- a/src/io/textfile_decoder.cc
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "singa/io/decoder.h"
-#include <string>
-#include <sstream>
-
-#define MAXSIZE 4096
-
-namespace singa {
-
-std::vector<Tensor> TextDecoder::Decode(std::string value) {
-  std::vector<Tensor> output;
-  std::stringstream ss;
-  ss.str(value);
-  int l = 0;
-  if (has_label_ == true) ss >> l;
-  std::string str;
-  float* d = new float[MAXSIZE];
-  size_t size = 0;
-  while(std::getline(ss, str, ',')) {
-    float temp;
-    if (std::stringstream(str) >> temp) {
-      CHECK_LE(size, MAXSIZE-1);
-      d[size++] = temp;
-    }
-  }
-
-  Tensor data(Shape{size}, kFloat32);
-  data.CopyDataFromHostPtr(d, size);
-  output.push_back(data);
-  if (has_label_ == true) {
-    Tensor label(Shape{1}, kInt);
-    label.CopyDataFromHostPtr(&l, 1);
-    output.push_back(label);
-  }
-  return output;
-}
-}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/src/io/textfile_encoder.cc
----------------------------------------------------------------------
diff --git a/src/io/textfile_encoder.cc b/src/io/textfile_encoder.cc
deleted file mode 100644
index 72401ee..0000000
--- a/src/io/textfile_encoder.cc
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "singa/io/encoder.h"
-#include <sstream>
-
-namespace singa {
-
-std::string TextEncoder::Encode(vector<Tensor>& data) {
-  CHECK_GE(data.size(), 1);
-  size_t size = data[0].Size();
-  const float* value = data[0].data<float>();
-  std::string des = "";
-  if (data.size() == 2) {
-    const float label = (const float)data[1].data<int>()[0];
-    std::ostringstream buff;
-    buff << label;
-    des += buff.str() + ',';
-  }
-  for (size_t i = 0; i < size; i++) {
-    std::ostringstream buff;
-    buff << value[i];
-    if (i == size - 1) des += buff.str();
-    else des += buff.str() + ',';
-  }
-  return des;
-}
-}  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_csv.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_csv.cc b/test/singa/test_csv.cc
new file mode 100644
index 0000000..d8dbe69
--- /dev/null
+++ b/test/singa/test_csv.cc
@@ -0,0 +1,60 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/io/encoder.h"
+#include "singa/io/decoder.h"
+#include "gtest/gtest.h"
+#include <sstream>
+#include <algorithm>
+
+using singa::Shape;
+using singa::Tensor;
+TEST(CSV, EncoderDecode) {
+  singa::CSVEncoder encoder;
+  singa::CSVDecoder decoder;
+
+  singa::DecoderConf decoder_conf;
+  decoder_conf.set_has_label(true);
+  decoder.Setup(decoder_conf);
+  EXPECT_EQ(true, decoder.has_label());
+
+  float in_data[] = {1.23, 4.5, 5.1, 3.33, 0.44};
+  std::string in_str = "2, 1.23, 4.5, 5.1, 3.33, 0.44";
+  int in_label = 2;
+  size_t size = 5;
+
+  std::vector<Tensor> input;
+  Tensor data(Shape{size}, singa::kFloat32), label(Shape{1}, singa::kInt);
+  data.CopyDataFromHostPtr<float>(in_data, size);
+  label.CopyDataFromHostPtr<int>(&in_label, 1);
+  input.push_back(data);
+  input.push_back(label);
+
+  std::string value = encoder.Encode(input);
+  in_str.erase(std::remove(in_str.begin(), in_str.end(), ' '), in_str.end());
+  EXPECT_EQ(in_str, value);
+
+  std::vector<Tensor> output = decoder.Decode(value);
+  const auto* out_data = output.at(0).data<float>();
+  const auto* out_label = output.at(1).data<int>();
+  for (size_t i = 0; i < size; i++) EXPECT_EQ(in_data[i], out_data[i]);
+  EXPECT_EQ(in_label, out_label[0]);
+}

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_decoder.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_decoder.cc b/test/singa/test_decoder.cc
deleted file mode 100644
index fa8683f..0000000
--- a/test/singa/test_decoder.cc
+++ /dev/null
@@ -1,84 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied.  See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/io/encoder.h"
-#include "singa/io/decoder.h"
-#include "gtest/gtest.h"
-#include <time.h>
-
-#ifdef USE_OPENCV
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
-using singa::Shape;
-using singa::Tensor;
-TEST(Decoder, Decode) {
-  singa::JPG2ProtoEncoder encoder;
-  singa::Proto2JPGDecoder decoder;
-
-  // initial random seed
-  srand(time(NULL));
-
-  size_t height = 40, width = 30;
-  size_t nheight = 256, nwidth = 256, channel = 3;
-  size_t total = nheight * nwidth * channel;
-  cv::Mat image(height, width, CV_8UC3);
-  for (size_t i = 0; i < height; i++)
-    for (size_t j = 0; j < width; j++)
-      for (size_t k = 0; k < channel; k++)
-        image.at<cv::Vec3b>(i, j)[k] = static_cast<uchar>(rand() % 256);
-
-  cv::Mat transformed;
-  cv::Size size(nwidth, nheight);
-  cv::resize(image, transformed, size);
-  EXPECT_EQ(static_cast<int>(nwidth), transformed.cols);
-  EXPECT_EQ(static_cast<int>(nheight), transformed.rows);
-  EXPECT_EQ(static_cast<int>(channel), transformed.channels());
-
-  unsigned char* buff = transformed.data;
-  Shape shape{nheight, nwidth, channel};
-  Tensor pixel(shape, singa::kUChar), label(Shape{1}, singa::kInt);
-  pixel.CopyDataFromHostPtr<unsigned char>(buff, total);
-  int raw_label = 2;
-  label.CopyDataFromHostPtr<int>(&raw_label, 1);
-
-  std::vector<Tensor> input;
-  input.push_back(pixel);
-  input.push_back(label);
-  const auto* in_pixel = input[0].data<unsigned char>();
-  for (size_t i = 0; i < total; i++) EXPECT_EQ(buff[i], in_pixel[i]);
-  const int* in_label = input[1].data<int>();
-  EXPECT_EQ(2, in_label[0]);
-  EXPECT_EQ(2u, input.size());
-
-  std::string tmp = encoder.Encode(input);
-  std::vector<Tensor> output = decoder.Decode(tmp);
-  EXPECT_EQ(2u, output.size());
-  EXPECT_EQ(singa::kFloat32, output[0].data_type());
-  Shape out_shape = output[0].shape();
-  for (size_t i = 0; i < shape.size(); i++) EXPECT_EQ(shape[i], out_shape[i]);
-  const int* out_label = output[1].data<int>();
-  EXPECT_EQ(raw_label, out_label[0]);
-  // opencv imencode will have some information loss
-  // const float* out_pixel = output[0].data<const float*>();
-  // for(size_t i = 0; i < total; i++)
-  //  EXPECT_LE(fabs(in_pixel[i]-out_pixel[i]), 10.f);
-}
-#endif

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_jpg.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_jpg.cc b/test/singa/test_jpg.cc
new file mode 100644
index 0000000..f60546a
--- /dev/null
+++ b/test/singa/test_jpg.cc
@@ -0,0 +1,84 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/io/encoder.h"
+#include "singa/io/decoder.h"
+#include "gtest/gtest.h"
+#include <time.h>
+
+#ifdef USE_OPENCV
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+using singa::Shape;
+using singa::Tensor;
+TEST(Decoder, Decode) {
+  singa::JPGEncoder encoder;
+  singa::JPGDecoder decoder;
+
+  // initial random seed
+  srand(time(NULL));
+
+  size_t height = 40, width = 30;
+  size_t nheight = 256, nwidth = 256, channel = 3;
+  size_t total = nheight * nwidth * channel;
+  cv::Mat image(height, width, CV_8UC3);
+  for (size_t i = 0; i < height; i++)
+    for (size_t j = 0; j < width; j++)
+      for (size_t k = 0; k < channel; k++)
+        image.at<cv::Vec3b>(i, j)[k] = static_cast<uchar>(rand() % 256);
+
+  cv::Mat transformed;
+  cv::Size size(nwidth, nheight);
+  cv::resize(image, transformed, size);
+  EXPECT_EQ(static_cast<int>(nwidth), transformed.cols);
+  EXPECT_EQ(static_cast<int>(nheight), transformed.rows);
+  EXPECT_EQ(static_cast<int>(channel), transformed.channels());
+
+  unsigned char* buff = transformed.data;
+  Shape shape{nheight, nwidth, channel};
+  Tensor pixel(shape, singa::kUChar), label(Shape{1}, singa::kInt);
+  pixel.CopyDataFromHostPtr<unsigned char>(buff, total);
+  int raw_label = 2;
+  label.CopyDataFromHostPtr<int>(&raw_label, 1);
+
+  std::vector<Tensor> input;
+  input.push_back(pixel);
+  input.push_back(label);
+  const auto* in_pixel = input[0].data<unsigned char>();
+  for (size_t i = 0; i < total; i++) EXPECT_EQ(buff[i], in_pixel[i]);
+  const int* in_label = input[1].data<int>();
+  EXPECT_EQ(2, in_label[0]);
+  EXPECT_EQ(2u, input.size());
+
+  std::string tmp = encoder.Encode(input);
+  std::vector<Tensor> output = decoder.Decode(tmp);
+  EXPECT_EQ(2u, output.size());
+  EXPECT_EQ(singa::kFloat32, output[0].data_type());
+  Shape out_shape = output[0].shape();
+  for (size_t i = 0; i < shape.size(); i++) EXPECT_EQ(shape[i], out_shape[i]);
+  const int* out_label = output[1].data<int>();
+  EXPECT_EQ(raw_label, out_label[0]);
+  // opencv imencode will have some information loss
+  // const float* out_pixel = output[0].data<const float*>();
+  // for(size_t i = 0; i < total; i++)
+  //  EXPECT_LE(fabs(in_pixel[i]-out_pixel[i]), 10.f);
+}
+#endif

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/49319223/test/singa/test_textfile_decoder.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_textfile_decoder.cc b/test/singa/test_textfile_decoder.cc
deleted file mode 100644
index bb31b88..0000000
--- a/test/singa/test_textfile_decoder.cc
+++ /dev/null
@@ -1,60 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*   http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied.  See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/io/encoder.h"
-#include "singa/io/decoder.h"
-#include "gtest/gtest.h"
-#include <sstream>
-#include <algorithm>
-
-using singa::Shape;
-using singa::Tensor;
-TEST(TextDecoder, Decode) {
-  singa::TextEncoder encoder;
-  singa::TextDecoder decoder;
-
-  singa::DecoderConf decoder_conf;
-  decoder_conf.set_has_label(true);
-  decoder.Setup(decoder_conf);
-  EXPECT_EQ(true, decoder.has_label());
-
-  float in_data[] = {1.23, 4.5, 5.1, 3.33, 0.44};
-  std::string in_str = "2, 1.23, 4.5, 5.1, 3.33, 0.44";
-  int in_label = 2;
-  size_t size = 5;
-
-  std::vector<Tensor> input;
-  Tensor data(Shape{size}, singa::kFloat32), label(Shape{1}, singa::kInt);
-  data.CopyDataFromHostPtr<float>(in_data, size);
-  label.CopyDataFromHostPtr<int>(&in_label, 1);
-  input.push_back(data);
-  input.push_back(label);
-
-  std::string value = encoder.Encode(input);
-  in_str.erase(std::remove(in_str.begin(), in_str.end(), ' '), in_str.end());
-  EXPECT_EQ(in_str, value);
-
-  std::vector<Tensor> output = decoder.Decode(value);
-  const auto* out_data = output.at(0).data<float>();
-  const auto* out_label = output.at(1).data<int>();
-  for (size_t i = 0; i < size; i++) EXPECT_EQ(in_data[i], out_data[i]);
-  EXPECT_EQ(in_label, out_label[0]);
-}


Mime
View raw message