singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [3/7] incubator-singa git commit: SINGA-97 Add HDFS Store
Date Sat, 02 Jan 2016 15:20:04 GMT
SINGA-97 Add HDFS Store

Revert changes to the examples directory, assuming that users upload data
to HDFS manually.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/374f11d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/374f11d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/374f11d6

Branch: refs/heads/master
Commit: 374f11d62e83df6b681c36f9557b6825ab8c236a
Parents: 8a07a29
Author: Anh Dinh <ug93tad@gmail.com>
Authored: Mon Dec 28 15:51:40 2015 +0800
Committer: WANG Sheng <wangsheng1001@gmail.com>
Committed: Sat Jan 2 19:58:14 2016 +0800

----------------------------------------------------------------------
 Makefile.am                       |  2 +-
 examples/cifar10/Makefile.example |  5 -----
 examples/cifar10/create_data.cc   | 30 ++++++++++++------------------
 examples/mnist/Makefile.example   |  5 -----
 examples/mnist/create_data.cc     | 13 +------------
 include/singa/io/hdfsfile.h       |  2 +-
 src/io/kvfile_store.cc            |  4 +++-
 7 files changed, 18 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index d78a150..cc308f7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -33,7 +33,7 @@ CFLAGS += $(DEBUG)
 CXXFLAGS += $(DEBUG)
 AC_CXXFLAGS = $(DEBUG)
 
-INCLUDES = -I$(top_srcdir)/include -I/usr/local/include/hdfs
+INCLUDES = -I$(top_srcdir)/include -I/usr/local/include
 
 PROTOS := $(top_srcdir)/src/proto/singa.proto \
           $(top_srcdir)/src/proto/job.proto \

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/cifar10/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/cifar10/Makefile.example b/examples/cifar10/Makefile.example
index 775e165..dd65d7d 100644
--- a/examples/cifar10/Makefile.example
+++ b/examples/cifar10/Makefile.example
@@ -28,11 +28,6 @@ cifar-10-binary-bin:
 	wget http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz
 	tar xf cifar-10-binary.tar.gz
 
-compile:
-	$(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 \
-		-I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \
-		-Wl,-rpath=../../.libs/  -o create_data.bin
-
 create:
 	$(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \
 		-I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/cifar10/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/cifar10/create_data.cc b/examples/cifar10/create_data.cc
index 05169d8..5873c0e 100644
--- a/examples/cifar10/create_data.cc
+++ b/examples/cifar10/create_data.cc
@@ -19,16 +19,16 @@
 *
 *************************************************************/
 
-
-/**
- * Create training and test DataShard for CIFAR dataset. 
- * It is adapted from convert_cifar_data from Caffe. 
- *    create_shard.bin <input> <output_folder> 
- * 
- * Read from JobConf object the option to use KVfile, HDFS or other (1st layer
- * store_conf object). 
- * To load to HDFS, specify "hdfs://namenode/examples" as the output folder
- */
+//
+// This code creates training and test DataShard for CIFAR dataset.
+// It is adapted from the convert_cifar_data from Caffe
+//
+// Usage:
+//    create_shard.bin input_folder output_folder
+//
+// The CIFAR dataset could be downloaded at
+//    http://www.cs.toronto.edu/~kriz/cifar.html
+//
 
 #include <glog/logging.h>
 #include <fstream>
@@ -38,8 +38,6 @@
 
 #include "singa/io/store.h"
 #include "singa/proto/common.pb.h"
-#include "singa/utils/common.h"
-#include "singa/proto/job.pb.h"
 
 using std::string;
 
@@ -47,7 +45,6 @@ const int kCIFARSize = 32;
 const int kCIFARImageNBytes = 3072;
 const int kCIFARBatchSize = 10000;
 const int kCIFARTrainBatches = 5;
-const char JOB_CONFIG[] = "job.conf";
 
 void read_image(std::ifstream* file, int* label, char* buffer) {
   char label_char;
@@ -61,6 +58,7 @@ void create_data(const string& input_folder, const string& output_folder)
{
   int label;
   char str_buffer[kCIFARImageNBytes];
   string rec_buf;
+
   singa::RecordProto image;
   image.add_shape(3);
   image.add_shape(kCIFARSize);
@@ -71,11 +69,7 @@ void create_data(const string& input_folder, const string& output_folder)
{
   for (int i = 0; i < kCIFARImageNBytes; i++)
     mean.add_data(0.f);
 
-  singa::JobProto job_proto;
-  singa::ReadProtoFromTextFile(JOB_CONFIG, &job_proto);
-  string store_backend =
-        job_proto.neuralnet().layer(0).store_conf().backend();
-  auto store = singa::io::CreateStore(store_backend);
+  auto store = singa::io::CreateStore("kvfile");
   CHECK(store->Open(output_folder + "/train_data.bin", singa::io::kCreate));
   LOG(INFO) << "Preparing training data";
   int count = 0;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/mnist/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example
index ba2308b..733633d 100644
--- a/examples/mnist/Makefile.example
+++ b/examples/mnist/Makefile.example
@@ -33,11 +33,6 @@ mnist:
 	gunzip train-images-idx3-ubyte.gz && gunzip train-labels-idx1-ubyte.gz
 	gunzip t10k-images-idx3-ubyte.gz && gunzip t10k-labels-idx1-ubyte.gz
 
-compile:
-	$(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 -I../../include \
-		-L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
-		-o create_data.bin
-
 create:
 	$(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \
 		-L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/mnist/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc
index 66a4905..5e51e97 100644
--- a/examples/mnist/create_data.cc
+++ b/examples/mnist/create_data.cc
@@ -38,19 +38,14 @@
 #include "singa/io/store.h"
 #include "singa/utils/common.h"
 #include "singa/proto/common.pb.h"
-#include "singa/proto/job.pb.h"
 
 using std::string;
 
-const char JOB_CONFIG[] = "job.conf";
-
 uint32_t swap_endian(uint32_t val) {
     val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
     return (val << 16) | (val >> 16);
 }
 
-// output is the full path, unlike create_data in CIFAR with only
-// specifies the directory
 void create_data(const char* image_filename, const char* label_filename,
         const char* output) {
   // Open files
@@ -81,13 +76,7 @@ void create_data(const char* image_filename, const char* label_filename,
   image_file.read(reinterpret_cast<char*>(&cols), 4);
   cols = swap_endian(cols);
 
-  // read backend from the job.conf
-  singa::JobProto job_proto;
-  singa::ReadProtoFromTextFile(JOB_CONFIG.c_str(), &job_proto);
-  string store_backend =
-    job_proto.neuralnet().layer(0).store_conf().backend();
-
-  auto store = singa::io::OpenStore(store_backend, output, singa::io::kCreate);
+  auto store = singa::io::OpenStore("kvfile", output, singa::io::kCreate);
   char label;
   char* pixels = new char[rows * cols];
   int count = 0;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/include/singa/io/hdfsfile.h
----------------------------------------------------------------------
diff --git a/include/singa/io/hdfsfile.h b/include/singa/io/hdfsfile.h
index f92910e..cd3ded3 100644
--- a/include/singa/io/hdfsfile.h
+++ b/include/singa/io/hdfsfile.h
@@ -33,7 +33,7 @@
 #include <google/protobuf/message.h>
 #endif
 
-#include <hdfs.h>
+#include <hdfs/hdfs.h>
 
 namespace singa {
 namespace io {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/src/io/kvfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc
index 79e2a40..e77f49b 100644
--- a/src/io/kvfile_store.cc
+++ b/src/io/kvfile_store.cc
@@ -56,7 +56,9 @@ void KVFileStore::SeekToFirst() {
   file_->SeekToFirst();
 }
 
-void KVFileStore::Seek(int offset){}
+void KVFileStore::Seek(int offset){
+  LOG(FATAL) << "Operation not supported.";
+}
 
 bool KVFileStore::Write(const std::string& key, const std::string& value) {
   CHECK_NE(mode_, kRead);


Mime
View raw message