singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wang...@apache.org
Subject [2/2] incubator-singa git commit: SINGA-155 Remove zookeeper for single-process training
Date Sat, 02 Apr 2016 03:46:35 GMT
SINGA-155 Remove zookeeper for single-process training

make zookeeper optional, configure with --enable-zookeeper to use zookeeper
  * add USE_ZOOKEEPER flags
  * update makefile
  * check cpplint


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/0233049c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/0233049c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/0233049c

Branch: refs/heads/master
Commit: 0233049ce13d3f430affe4e32a88cbaf4a3e17e4
Parents: adeced6
Author: WANG Sheng <wangsheng1001@gmail.com>
Authored: Thu Mar 31 22:03:42 2016 +0800
Committer: WANG Sheng <wangsheng1001@gmail.com>
Committed: Fri Apr 1 12:59:59 2016 +0800

----------------------------------------------------------------------
 Makefile.am                      | 94 ++++++++++++++++++++--------------
 configure.ac                     | 95 +++++++++++++++++++----------------
 include/singa/utils/cluster_rt.h |  3 ++
 src/utils/cluster.cc             |  3 +-
 src/utils/cluster_rt.cc          | 12 +++++
 5 files changed, 124 insertions(+), 83 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index f185a9d..95f7e81 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -32,19 +32,22 @@ CUDA_OBJS := src/utils/math_kernel.o
 CUDA_HDRS := include/singa/utils/math_kernel.h
 
 CUDNN_SRCS := src/neuralnet/loss_layer/cudnn_softmaxloss.cc \
-			  src/neuralnet/neuron_layer/cudnn_softmax.cc \
-			  src/neuralnet/neuron_layer/cudnn_pooling.cc \
-			  src/neuralnet/neuron_layer/cudnn_activation.cc \
-			  src/neuralnet/neuron_layer/cudnn_lrn.cc \
-			  src/neuralnet/neuron_layer/cudnn_convolution.cc
+              src/neuralnet/neuron_layer/cudnn_softmax.cc \
+              src/neuralnet/neuron_layer/cudnn_pooling.cc \
+              src/neuralnet/neuron_layer/cudnn_activation.cc \
+              src/neuralnet/neuron_layer/cudnn_lrn.cc \
+              src/neuralnet/neuron_layer/cudnn_convolution.cc
 
 PY_SRCS := tool/python/singa/driver_wrap.cxx \
-		   src/driver.cc
+           src/driver.cc
+
+ZOOKEEPER_SRCS := src/utils/zk_service.cc
+ZOOKEEPER_HDRS := include/singa/utils/zk_service.h
 
 HDFS_SRCS := src/io/hdfsfile.cc \
-			 src/io/hdfsfile_store.cc
+             src/io/hdfsfile_store.cc
 HDFS_HDRS := include/singa/io/hdfsfile.h \
-			 include/singa/io/hdfsfile_store.h
+             include/singa/io/hdfsfile_store.h
 
 SINGA_SRCS := src/driver.cc \
               src/server.cc \
@@ -101,14 +104,13 @@ SINGA_SRCS := src/driver.cc \
               src/utils/updater.cc \
               src/utils/blob.cc \
               src/utils/image_transform.cc \
-              src/utils/job_manager.cc \
-              src/utils/zk_service.cc 
+              src/utils/job_manager.cc 
 
 
 SINGA_HDRS := include/singa.h \
-			  include/singa/utils/math_blob.h \
-			  include/singa/utils/math_addr.h \
-			  include/singa/utils/cluster.h \
+              include/singa/utils/math_blob.h \
+              include/singa/utils/math_addr.h \
+              include/singa/utils/cluster.h \
               include/utils/cluster_rt.h \
               include/utils/param.h \
               include/utils/common.h \
@@ -121,8 +123,7 @@ SINGA_HDRS := include/singa.h \
               include/utils/tinydir.h \
               include/utils/tokenizer.h \
               include/utils/image_transform.h \
-              include/utils/job_manager.cc \
-              include/utils/zk_service.cc \
+              include/utils/job_manager.h \
               include/server.h \
               include/worker.h \
               include/stub.h \
@@ -151,19 +152,19 @@ SINGA_HDRS := include/singa.h \
 GTEST_SRCS := include/gtest/gtest-all.cc
 GTEST_HRDS := include/gtest/gtest.h
 TEST_SRCS := include/gtest/gtest_main.cc \
-						 src/test/test_cluster.cc \
-						 src/test/test_common.cc \
-						 src/test/test_msg.cc \
-						 src/test/test_math.cc \
-						 src/test/test_neuralnet.cc \
-						 src/test/test_paramslicer.cc \
-						 src/test/test_kvfile.cc \
-						 src/test/test_store.cc \
-						 src/test/test_connection_layers.cc \
-						 src/test/test_record_input_layer.cc \
-						 src/test/test_csv_input_layer.cc \
-             					 src/test/test_gru_layer.cc \
-						 src/test/test_unrolling.cc
+             src/test/test_cluster.cc \
+             src/test/test_common.cc \
+             src/test/test_msg.cc \
+             src/test/test_math.cc \
+             src/test/test_neuralnet.cc \
+             src/test/test_paramslicer.cc \
+             src/test/test_kvfile.cc \
+             src/test/test_store.cc \
+             src/test/test_connection_layers.cc \
+             src/test/test_record_input_layer.cc \
+             src/test/test_csv_input_layer.cc \
+             src/test/test_gru_layer.cc \
+             src/test/test_unrolling.cc
 
 #EXTRA_PROGRAMS = $(PROGS)
 EXTRA_PROGRAMS = singatest test
@@ -194,6 +195,11 @@ libsinga_la_SOURCES += $(CUDNN_SRCS)
 libsinga_la_CXXFLAGS += $(CUDNN_CFLAGS)
 libsinga_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
 endif
+if DZOOKEEPER
+libsinga_la_SOURCES += $(ZOOKEEPER_SRCS)
+libsinga_la_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+libsinga_la_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
+endif
 
 if DHDFS
 libsinga_la_SOURCES += $(HDFS_SRCS)
@@ -209,8 +215,7 @@ singa_LDFLAGS = -lsinga \
                 -lprotobuf \
                 -lopenblas \
                 -lzmq \
-                -lczmq \
-                -lzookeeper_mt
+                -lczmq 
 if LMDB
 singa_LDFLAGS += -llmdb
 endif
@@ -221,6 +226,12 @@ singa_CXXFLAGS += $(CUDA_CFLAGS)
 singa_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
 endif
 
+if DZOOKEEPER
+singa_SOURCES += $(ZOOKEEPER_SRCS)
+singa_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+singa_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
+endif
+
 if DCUDNN
 singa_SOURCES += $(CUDNN_SRCS)
 singa_CXXFLAGS += $(CUDNN_CFLAGS)
@@ -235,17 +246,17 @@ endif
 #bin_PROGRAMS += singatool
 singatool_SOURCES = src/utils/tool.cc #$(CUDA_SRCS) $(CUDA_HDRS) $(CUDNN_SRCS)
 singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas \
-					 -funroll-loops -DTHREADED -I$(top_srcdir)/include $(DEFAULT_FLAGS)
+              -funroll-loops -DTHREADED -I$(top_srcdir)/include $(DEFAULT_FLAGS)
 singatool_LDFLAGS = -lsinga \
                     -lglog  \
-                    -lprotobuf \
-                    -lzookeeper_mt
+                    -lprotobuf 
 
-if DHDFS
-singatool_SOURCES += $(HDFS_SRCS)
-singatool_CXXFLAGS += $(HDFS_CFLAGS)
-singatool_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+if DZOOKEEPER
+singatool_SOURCES += $(ZOOKEEPER_SRCS)
+singatool_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+singatool_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
 endif
+
 #lib_LTLIBRARIES += libgtest.la
 libgtest_la_SOURCES = $(GTEST_HDRS) $(GTEST_SRCS)
 libgtest_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include
@@ -265,8 +276,7 @@ singatest_LDFLAGS = -lsinga \
                 -lopenblas \
                 -lzmq \
                 -lczmq \
-                -lzookeeper_mt \
-			    -lgtest
+                -lgtest
 if LMDB
 singatest_LDFLAGS += -llmdb
 endif
@@ -283,6 +293,12 @@ singatest_CXXFLAGS += $(CUDNN_CFLAGS)
 singatest_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
 endif
 
+if DZOOKEEPER
+singatest_SOURCES += $(ZOOKEEPER_SRCS)
+singatest_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+singatest_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
+endif
+
 _driver_la_SOURCES = $(PY_SRCS)
 _driver_la_CXXFLAGS = $(DEFAULT_FLAGS) $(MSHADOW_FLAGS) -I$(top_srcdir)/include $(PYFLAGS)
 _driver_la_LDFLAGS = -lsinga -module -shared $(PYLIBS) -avoid-version -rpath $(pydir)

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 41ff510..af80c1e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -36,13 +36,32 @@ AC_PROG_CC
 AC_DISABLE_STATIC
 AC_PROG_LIBTOOL
 
+# Checks for libraries.
+AC_SEARCH_LIBS([cblas_sgemm], [openblas], [], [
+  AC_MSG_ERROR([unable to find cblas_sgemm() function])
+  ])
+AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [
+  AC_MSG_ERROR([unable to find zmq_ctx_new() function])
+    ])
+AC_SEARCH_LIBS([zmsg_new], [czmq], [], [
+  AC_MSG_ERROR([unable to find zmsg_new() function])
+  ])
+AC_CHECK_LIB([glog], [main], [], [
+  AC_MSG_ERROR([unable to find glog library])
+  ])
+AC_CHECK_LIB([protobuf], [main], [], [
+  AC_MSG_ERROR([unable to find protobuf library])
+  ])
+
+PROGS=''
+LTLIBS=''
+
 # Setup custom CUDA paths
 AC_ARG_ENABLE(cuda,
   [AS_HELP_STRING(--enable-cuda,enable CUDA support)],
     cudaval="yes",
     cudaval="no")
 AM_CONDITIONAL(DCUDA, [test "$cudaval" = "yes"])
-
 AC_ARG_WITH([cuda],
    [AS_HELP_STRING(
         [--with-cuda=PATH],
@@ -54,7 +73,6 @@ if test "$cuda_prefix" == "yes"; then
         cuda_prefix="/usr/local/cuda"
     fi
 fi
-
 if test x"$cudaval" = x"yes"; then
     AC_MSG_CHECKING([nvcc in $cuda_prefix/bin])
     if test -x "$cuda_prefix/bin/nvcc"; then
@@ -100,7 +118,6 @@ AC_ARG_ENABLE([cudnn],
     [AS_HELP_STRING(--enable-cudnn,enable CUDNN support)],
     [enable_cudnn=yes], [enable_cudnn=no])
 AM_CONDITIONAL(DCUDNN, [test "$enable_cudnn" = "yes"])
-
 AC_ARG_WITH([cudnn],
     [AS_HELP_STRING([--with-cudnn=PATH], [prefix where CUDNN is installed])],
     [cudnn_prefix=$withval], [cudnn_prefix="/usr/local/cuda"])
@@ -109,7 +126,6 @@ if test "$cudnn_prefix" == "yes"; then
         cudnn_prefix="/usr/local/cuda"
     fi
 fi
-
 if test x"$enable_cudnn" == x"yes"; then
     CUDNN_CFLAGS="-I$cudnn_prefix/include"
     CUDNN_LDFLAGS="-L$cudnn_prefix/lib64 -L$cudnn_prefix/lib"
@@ -126,27 +142,42 @@ else
     CUDNN_LDFLAGS=""
     CUDNN_LIBS=""
 fi
-
 AC_SUBST(CUDNN_CFLAGS)
 AC_SUBST(CUDNN_LDFLAGS)
 AC_SUBST(CUDNN_LIBS)
 
-# Checks for libraries.
-AC_SEARCH_LIBS([cblas_sgemm], [openblas], [], [
-  AC_MSG_ERROR([unable to find cblas_sgemm() function])
-  ])
-AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [
-  AC_MSG_ERROR([unable to find zmq_ctx_new() function])
-    ])
-AC_SEARCH_LIBS([zmsg_new], [czmq], [], [
-  AC_MSG_ERROR([unable to find zmsg_new() function])
-  ])
-AC_CHECK_LIB([glog], [main], [], [
-  AC_MSG_ERROR([unable to find glog library])
-  ])
-AC_CHECK_LIB([protobuf], [main], [], [
-  AC_MSG_ERROR([unable to find protobuf library])
-  ])
+# Setup custom zookeeper paths 
+AC_ARG_ENABLE(zookeeper,
+  AS_HELP_STRING([--enable-zookeeper],[enable zookeeper support]),
+  [enable_zookeeper=yes],[enable_zookeeper=no])
+AM_CONDITIONAL(DZOOKEEPER, test "$enable_zookeeper" = yes)
+AC_ARG_WITH([zookeeper],
+    [AS_HELP_STRING([--with-zookeeper=PATH], [prefix where zookeeper is installed])],
+    [zookeeper_prefix=$withval], [zookeeper_prefix="/usr/local"])
+if test "$zookeeper_prefix" == "yes"; then
+    if test "$withval" == "yes"; then
+        cudnn_prefix="/usr/local"
+    fi
+fi
+if test x"$enable_zookeeper" != x"no"; then
+  ZOOKEEPER_CFLAGS="-I$zookeeper_prefix/include"
+  ZOOKEEPER_LDFLAGS="-L$zookeeper_prefix/lib"
+  ZOOKEEPER_LIBS="-lzookeeper_mt"
+  LIBS="$LIBS $ZOOKEEPER_LIBS"
+  LDFLAGS="$LDFLAGS $ZOOKEEPER_LDFLAGS"
+  DEBUG+=" -DUSE_ZOOKEEPER"
+  AC_DEFINE(DZOOKEEPER,[1],[Defined if zookeeper should be used])
+  AC_CHECK_LIB([zookeeper_mt], [main], [], [
+      AC_MSG_ERROR([unable to find zookeeper library])
+      ])
+else
+  ZOOKEEPER_CFLAGS=""
+  ZOOKEEPER_LDFLAGS=""
+  ZOOKEEPER_LIBS=""
+fi
+AC_SUBST(ZOOKEEPER_CFLAGS)
+AC_SUBST(ZOOKEEPER_LDFLAGS)
+AC_SUBST(ZOOKEEPER_LIBS)
 
 # Setup custom lmdb paths 
 AC_ARG_ENABLE(lmdb,
@@ -160,14 +191,11 @@ if test x"$enable_lmdb" = x"yes"; then
   AC_DEFINE(LMDB, 1, [Enable Option layer])
 fi
 
-PROGS=''
-LTLIBS=''
 # Setup custom libhdfs paths 
 AC_ARG_ENABLE(hdfs,
   AS_HELP_STRING([--enable-hdfs],[enable hdfs support]),
   [enable_hdfs=yes],[enable_hdfs=no])
 AM_CONDITIONAL(DHDFS, test "$enable_hdfs" = yes)
-
 AC_ARG_WITH([libhdfs],
     [AS_HELP_STRING([--with-libhdfs=PATH], [prefix where libhdfs is installed])],
     [hdfs_prefix=$withval], [hdfs_prefix="/usr/local"])
@@ -176,7 +204,6 @@ if test "$hdfs_prefix" == "yes"; then
         cudnn_prefix="/usr/local"
     fi
 fi
-
 if test x"$enable_hdfs" != x"no"; then
   HDFS_CFLAGS="-I$hdfs_prefix/include"
   HDFS_LDFLAGS="-L$hdfs_prefix/lib"
@@ -193,7 +220,6 @@ else
   HDFS_LDFLAGS=""
   HDFS_LIBS=""
 fi
-
 AC_SUBST(HDFS_CFLAGS)
 AC_SUBST(HDFS_LDFLAGS)
 AC_SUBST(HDFS_LIBS)
@@ -246,8 +272,6 @@ if test "$python_prefix" == "yes"; then
         python_prefix="/usr/include/python`python -V 2>&1 | awk '{print substr($2,1,3)}'`"
     fi
 fi
-
-
 if test x"$enable_python" != x"no"; then
     AC_MSG_CHECKING([Python.h in $python_prefix])
     if test -f "$python_prefix/Python.h"; then
@@ -264,27 +288,12 @@ else
     PY_PROGS=''
     PYFLAGS=''
 fi
-
 AC_SUBST([PROGS])
 AC_SUBST([LTLIBS])
 AC_SUBST([DEBUG])
 AC_SUBST([PYFLAGS])
 AC_SUBST([PY_PROGS])
 
-# Setup for opencv libs
-#AC_CHECK_LIB([opencv_imgproc], [main], [], [
-#  AC_MSG_ERROR([unable to find opencv_imgproc lib])
-#  ])
-#AC_CHECK_LIB([opencv_highgui], [main], [], [
-#  AC_MSG_ERROR([unable to find opencv_highgui lib])
-#  ])
-#AC_CHECK_LIB([opencv_core], [main], [], [
-#  AC_MSG_ERROR([unable to find opencv_core lib])
-#  ])
-AC_CHECK_LIB([zookeeper_mt], [main], [], [
-  AC_MSG_ERROR([unable to find zookeeper])
-  ])
-
 # Checks for header files.
 AC_HEADER_STDC
 AC_CHECK_HEADERS([fcntl.h malloc.h stdlib.h])

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/include/singa/utils/cluster_rt.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/cluster_rt.h b/include/singa/utils/cluster_rt.h
index e0125be..4ab48bd 100644
--- a/include/singa/utils/cluster_rt.h
+++ b/include/singa/utils/cluster_rt.h
@@ -44,6 +44,9 @@ struct RTCallback {
  */
 class ClusterRuntime {
  public:
+  // ClusterRuntime have different implementation determined when compiling
+  static ClusterRuntime* Create(const std::string&host, int job_id);
+
   virtual ~ClusterRuntime() {}
   /**
    * Initialize the runtime instance

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/src/utils/cluster.cc
----------------------------------------------------------------------
diff --git a/src/utils/cluster.cc b/src/utils/cluster.cc
index 53c0444..13729f2 100644
--- a/src/utils/cluster.cc
+++ b/src/utils/cluster.cc
@@ -82,7 +82,8 @@ void Cluster::Init(int job, const SingaProto& singaConf,
     }
   }
   // cluster_rt_ = new ZKClusterRT(singa_.zookeeper_host(), job);
-  cluster_rt_ = new SPClusterRT();
+  // cluster_rt_ = new SPClusterRT();
+  cluster_rt_ = ClusterRuntime::Create(singa_.zookeeper_host(), job);
   cluster_rt_->Init();
   hostip_ = GetHostIP();
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/src/utils/cluster_rt.cc
----------------------------------------------------------------------
diff --git a/src/utils/cluster_rt.cc b/src/utils/cluster_rt.cc
index 33e04dc..7a04ff7 100644
--- a/src/utils/cluster_rt.cc
+++ b/src/utils/cluster_rt.cc
@@ -29,12 +29,24 @@
 #include <iostream>
 #include "singa/proto/job.pb.h"
 
+#ifdef USE_ZOOKEEPER
+#include "singa/utils/zk_service.h"
+#endif
+
 using std::string;
 using std::to_string;
 using std::vector;
 
 namespace singa {
 
+ClusterRuntime* ClusterRuntime::Create(const std::string&host, int job_id) {
+#ifdef USE_ZOOKEEPER
+  return new ZKClusterRT(host, job_id);
+#else
+  return new SPClusterRT();
+#endif
+}
+
 SPClusterRT::~SPClusterRT() {
   // release callback vector
   for (auto list : grp_callbacks_)


Mime
View raw message