singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [5/7] incubator-singa git commit: SINGA-97 Add HDFS Store
Date Sat, 02 Jan 2016 15:20:06 GMT
SINGA-97 Add HDFS Store

Modify compilation files. Now as a user, one can build SINGA with hdfs support by running:
	./configure --enable-hdfs --with-libhdfs=/PATH/TO/HDFS3
--with-libhdfs is optional as by default the path is /usr/local/.wq


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/9fbc8ee7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/9fbc8ee7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/9fbc8ee7

Branch: refs/heads/master
Commit: 9fbc8ee7aabbbdc2f76cdcccdf346e14d4544f1a
Parents: 374f11d
Author: xiezl <xiezhongle@comp.nus.edu.sg>
Authored: Thu Dec 31 15:25:01 2015 +0800
Committer: WANG Sheng <wangsheng1001@gmail.com>
Committed: Sat Jan 2 19:58:14 2016 +0800

----------------------------------------------------------------------
 Makefile.am  | 36 ++++++++++++++++++++++++++----------
 configure.ac | 50 +++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 69 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9fbc8ee7/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index cc308f7..a4315c8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -29,11 +29,11 @@ MSHADOW_FLAGS = -DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
 DEFAULT_FLAGS = -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
               $(MSHADOW_FLAGS) -funroll-loops -DTHREADED
 
-CFLAGS += $(DEBUG)
-CXXFLAGS += $(DEBUG)
-AC_CXXFLAGS = $(DEBUG)
+CFLAGS = $(DEBUG)
+CXXFLAGS = $(DEBUG)
+#AC_CXXFLAGS = $(DEBUG)
 
-INCLUDES = -I$(top_srcdir)/include -I/usr/local/include
+INCLUDES = -I$(top_srcdir)/include 
 
 PROTOS := $(top_srcdir)/src/proto/singa.proto \
           $(top_srcdir)/src/proto/job.proto \
@@ -62,6 +62,11 @@ CUDNN_SRCS := src/neuralnet/loss_layer/cudnn_softmaxloss.cc \
 PY_SRCS := tool/python/singa/driver_wrap.cxx \
 		   src/driver.cc
 
+HDFS_SRCS := src/io/hdfsfile.cc \
+			 src/io/hdfsfile_store.cc 
+HDFS_HDRS := include/singa/io/hdfsfile.h \
+			 include/singa/io/hdfsfile_store.h 
+
 SINGA_SRCS := src/driver.cc \
               src/server.cc \
               src/worker.cc \
@@ -102,8 +107,6 @@ SINGA_SRCS := src/driver.cc \
               src/io/kvfile_store.cc \
               src/io/textfile_store.cc \
               src/io/store.cc \
-              src/io/hdfsfile.cc \
-              src/io/hdfsfile_store.cc \
               src/utils/cluster.cc \
               src/utils/cluster_rt.cc \
               src/utils/graph.cc \
@@ -202,6 +205,11 @@ libsinga_la_CXXFLAGS += $(CUDNN_CFLAGS)
 libsinga_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
 endif
 
+if DHDFS
+libsinga_la_SOURCES += $(HDFS_SRCS)
+libsinga_la_CXXFLAGS += $(HDFS_CFLAGS)
+libsinga_la_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+endif
 
 #bin_PROGRAMS = singa
 singa_SOURCES = src/main.cc
@@ -213,8 +221,7 @@ singa_LDFLAGS = -lsinga \
                 -lopenblas \
                 -lzmq \
                 -lczmq \
-                -lzookeeper_mt \
-                -lhdfs3
+                -lzookeeper_mt 
 if LMDB
 singa_LDFLAGS += -llmdb
 endif
@@ -231,6 +238,11 @@ singa_CXXFLAGS += $(CUDNN_CFLAGS)
 singa_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
 endif
 
+if DHDFS
+singa_SOURCES += $(HDFS_SRCS)
+singa_CXXFLAGS += $(HDFS_CFLAGS)
+singa_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+endif
 #bin_PROGRAMS += singatool
 singatool_SOURCES = src/utils/tool.cc #$(CUDA_SRCS) $(CUDA_HDRS) $(CUDNN_SRCS)
 singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas \
@@ -238,8 +250,7 @@ singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas
\
 singatool_LDFLAGS = -lsinga \
                     -lglog  \
                     -lprotobuf \
-                    -lzookeeper_mt \
-                    -lhdfs3
+                    -lzookeeper_mt 
 
 #if DCUDA
 #singatool_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)  
@@ -253,6 +264,11 @@ singatool_LDFLAGS = -lsinga \
 #singatool_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
 #endif
 
+if DHDFS
+singatool_SOURCES += $(HDFS_SRCS)
+singatool_CXXFLAGS += $(HDFS_CFLAGS)
+singatool_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+endif
 #lib_LTLIBRARIES += libgtest.la
 libgtest_la_SOURCES = $(GTEST_HDRS) $(GTEST_SRCS)
 libgtest_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9fbc8ee7/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index bde1d8e..82db55a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -55,7 +55,6 @@ if test "$cuda_prefix" == "yes"; then
     fi
 fi
 
-
 if test x"$cudaval" = x"yes"; then
     AC_MSG_CHECKING([nvcc in $cuda_prefix/bin])
     if test -x "$cuda_prefix/bin/nvcc"; then
@@ -158,23 +157,60 @@ if test x"$enable_lmdb" = x"yes"; then
   AC_DEFINE(LMDB, 1, [Enable Option layer])
 fi
 
+PROGS=''
+LTLIBS=''
+AC_ARG_ENABLE(hdfs,
+  AS_HELP_STRING([--enable-hdfs],[enable hdfs support]),
+  [enable_hdfs=yes],[enable_hdfs=no])
+AM_CONDITIONAL(DHDFS, test "$enable_hdfs" = yes)
+
+AC_ARG_WITH([libhdfs],
+    [AS_HELP_STRING([--with-libhdfs=PATH], [prefix where libhdfs is installed])],
+    [hdfs_prefix=$withval], [hdfs_prefix="/usr/local"])
+if test "$hdfs_prefix" == "yes"; then
+    if test "$withval" == "yes"; then
+        cudnn_prefix="/usr/local"
+    fi
+fi
+
+if test x"$enable_hdfs" != x"no"; then
+  HDFS_CFLAGS="-I$hdfs_prefix/include"
+  HDFS_LDFLAGS="-L$hdfs_prefix/lib"
+  HDFS_LIBS="-lhdfs3"
+  LIBS="$LIBS $HDFS_LIBS"
+  LDFLAGS="$LDFLAGS $HDFS_LDFLAGS"
+  DEBUG+=" -DUSE_HDFS"
+  AC_DEFINE(DHDFS,[1],[Defined if HDFS should be used])
+  AC_CHECK_LIB([hdfs3], [main], [], [
+      AC_MSG_ERROR([unable to find hdfs3 library])
+      ])
+else
+  HDFS_CFLAGS=""
+  HDFS_LDFLAGS=""
+  HDFS_LIBS=""
+fi
+
+AC_SUBST(HDFS_CFLAGS)
+AC_SUBST(HDFS_LDFLAGS)
+AC_SUBST(HDFS_LIBS)
+
 AC_ARG_ENABLE(test,
   AS_HELP_STRING([--enable-test],[enable singa test]),
   [enable_test=yes],[enable_test=no])
 AM_CONDITIONAL(SINGATEST, test "$enable_test" = yes)
 if test x"$enable_test" != x"no"; then
-  PROGS='singatest test '
-  LTLIBS='libgtest.la '
+  PROGS+='singatest test '
+  LTLIBS+='libgtest.la '
 else
-  PROGS=''
-  LTLIBS=''
+  PROGS+=''
+  LTLIBS+=''
 fi
 
 AC_ARG_ENABLE(debug,
   AS_HELP_STRING([--enable-debug],[enable debug mode]),
   [enable_debug=yes],[enable_debug=no])
-AM_CONDITIONAL(DEBUG, test "$enable_debug" = yes)
-if test x"$enable_debug" != x"no"; then
+AM_CONDITIONAL(SINGADEBUG, [test "$enable_debug" = yes])
+if test x"$enable_debug" == x"yes"; then
   DEBUG+=' -g'
 else
   DEBUG+=' -O2'


Mime
View raw message