singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [1/7] incubator-singa git commit: SINGA-59 Remove dependency on gflags
Date Tue, 18 Aug 2015 03:02:10 GMT
Repository: incubator-singa
Updated Branches:
  refs/heads/master 2498ff135 -> f2b0aef12


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 277eb96..35c6d61 100644
--- a/configure.ac
+++ b/configure.ac
@@ -26,9 +26,6 @@ AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [
 AC_SEARCH_LIBS([zmsg_new], [czmq], [], [
   AC_MSG_ERROR([unable to find zmsg_new() function])
   ])
-AC_CHECK_LIB([gflags], [main], [], [
-  AC_MSG_ERROR([unable to find gflags library])
-  ])
 AC_CHECK_LIB([glog], [main], [], [
   AC_MSG_ERROR([unable to find glog library])
   ])

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/examples/mnist/create_shard.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_shard.cc b/examples/mnist/create_shard.cc
index 09229ac..635f901 100644
--- a/examples/mnist/create_shard.cc
+++ b/examples/mnist/create_shard.cc
@@ -7,7 +7,6 @@
 // The MNIST dataset could be downloaded at
 //    http://yann.lecun.com/exdb/mnist/
 
-#include <gflags/gflags.h>
 #include <glog/logging.h>
 #include <cstdint>
 #include <iostream>

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/include/singa.h
----------------------------------------------------------------------
diff --git a/include/singa.h b/include/singa.h
index 52d1f90..20f941d 100644
--- a/include/singa.h
+++ b/include/singa.h
@@ -2,7 +2,6 @@
 #define SINGA_SINGA_H_
 
 #include <cblas.h>
-#include <gflags/gflags.h>
 #include <glog/logging.h>
 #include <string>
 #include "communication/socket.h"
@@ -78,9 +77,16 @@ class Driver {
    * launching script.
    */
   inline int job_id() const { return job_id_; }
+  /**
+   * @return job conf path which is passed by users at the command line. It
+   * should at least contains the cluster configuration.
+   */
+  inline JobProto job_conf() const { return job_conf_; }
 
  private:
   int job_id_;
+  JobProto job_conf_;
+  SingaProto singa_conf_;
 };
 
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/include/utils/common.h
----------------------------------------------------------------------
diff --git a/include/utils/common.h b/include/utils/common.h
index c4ff4eb..acdb20e 100644
--- a/include/utils/common.h
+++ b/include/utils/common.h
@@ -2,7 +2,6 @@
 #define SINGA_UTILS_COMMON_H_
 
 #include <google/protobuf/message.h>
-#include <gflags/gflags.h>
 #include <stdlib.h>
 #include <unordered_map>
 #include <sstream>
@@ -10,11 +9,6 @@
 #include <vector>
 #include "proto/common.pb.h"
 
-#ifndef GFLAGS_GFLAGS_H_
-  namespace gflags = google;
-#endif  // GFLAGS_GFLAGS_H_
-
-
 namespace singa {
 using std::vector;
 
@@ -30,6 +24,16 @@ void ReadProtoFromBinaryFile(const char* filename,
 void WriteProtoToBinaryFile(const google::protobuf::Message& proto,
                             const char* filename);
 
+/**
+ * Locate the position of the arg in arglist.
+ *
+ * @param argc total num of arguments
+ * @param arglist all arguments
+ * @param the searched argument
+ * @return the position of arg in the arglist; -1 if not found.
+ */
+int ArgPos(int argc, char** arglist, const char* arg);
+
 const std::string CurrentDateTime();
 void  CreateFolder(const std::string name);
 /**

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 5469583..6d41fc5 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -2,17 +2,24 @@
 
 namespace singa {
 
-/**
- * the job and singa_conf arguments are passed by the singa script which is
- * transparent to users
- */
-DEFINE_int32(job, -1, "Unique job ID generated from singa-run.sh");
-DEFINE_string(singa_conf, "conf/singa.conf", "Global config file");
-
 void Driver::Init(int argc, char **argv) {
   google::InitGoogleLogging(argv[0]);
-  gflags::ParseCommandLineFlags(&argc, &argv, true);
-  job_id_ = FLAGS_job;
+
+  //  unique job ID generated from singa-run.sh, passed in as "-singa_job <id>"
+  int arg_pos = ArgPos(argc, argv, "-singa_job");
+  job_id_ = (arg_pos != -1) ? atoi(argv[arg_pos+1]) : -1;
+
+  //  global signa conf passed by singa-run.sh as "-singa_conf <path>"
+  arg_pos = ArgPos(argc, argv, "-singa_conf");
+  if (arg_pos != -1)
+    ReadProtoFromTextFile(argv[arg_pos+1], &singa_conf_);
+  else
+    ReadProtoFromTextFile("conf/singa.conf", &singa_conf_);
+
+  //  job conf passed by users as "-conf <path>"
+  arg_pos = ArgPos(argc, argv, "-conf");
+  CHECK_NE(arg_pos, -1);
+  ReadProtoFromTextFile(argv[arg_pos+1], &job_conf_);
 
   // register layers
   RegisterLayer<BridgeDstLayer>(kBridgeDst);
@@ -82,11 +89,9 @@ int Driver::RegisterWorker(int type) {
 }
 
 void Driver::Submit(bool resume, const JobProto& jobConf) {
-  SingaProto singaConf;
-  ReadProtoFromTextFile(FLAGS_singa_conf.c_str(), &singaConf);
-  if (singaConf.has_log_dir())
-    SetupLog(singaConf.log_dir(), std::to_string(FLAGS_job)
-        + "-" + jobConf.name());
+  if (singa_conf_.has_log_dir())
+    SetupLog(singa_conf_.log_dir(), std::to_string(job_id_)
+             + "-" + jobConf.name());
   if (jobConf.num_openblas_threads() != 1)
     LOG(WARNING) << "openblas with "
       << jobConf.num_openblas_threads() << " threads";
@@ -96,7 +101,7 @@ void Driver::Submit(bool resume, const JobProto& jobConf) {
   job.CopyFrom(jobConf);
   job.set_id(job_id_);
   Trainer trainer;
-  trainer.Start(resume, singaConf, &job);
+  trainer.Start(resume, singa_conf_, &job);
 }
 
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/main.cc
----------------------------------------------------------------------
diff --git a/src/main.cc b/src/main.cc
index 692fe2c..1f3f4a4 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -13,6 +13,9 @@
  * Optionally, users can register their own implemented classes, e.g., layer,
  * updater, through the registration func provided by the Driver.
  *
+ * Users must pass at least one argument to the singa-run.sh, i.e., the job
+ * configuration file which includes the cluster topology setting. Other fields
+ * e.g, neuralnet, updater can be configured in main.cc.
  *
  * TODO
  * Add helper functions for users to generate their configurations easily.
@@ -20,21 +23,21 @@
  * or, MLP(layer1_size, layer2_size, tanh, loss);
  */
 
-DEFINE_bool(resume, false, "Resume from checkpoint passed at cmd line");
-DEFINE_string(conf, "./job.conf", "job conf passed at cmd line");
-
 int main(int argc, char **argv) {
   //  must create driver at the beginning and call its Init method.
   singa::Driver driver;
   driver.Init(argc, argv);
 
+  //  if -resume in argument list, set resume to true; otherwise false
+  int resume_pos = singa::ArgPos(argc, argv, "-resume");
+  bool resume = (resume_pos != -1);
+
   //  users can register new subclasses of layer, updater, etc.
 
-  //  prepare job conf;
-  singa::JobProto jobConf;
-  singa::ReadProtoFromTextFile(FLAGS_conf.c_str(), &jobConf);
+  //  get the job conf, and custmize it if need
+  singa::JobProto jobConf = driver.job_conf();
 
   //  submit the job
-  driver.Submit(FLAGS_resume, jobConf);
+  driver.Submit(resume, jobConf);
   return 0;
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/utils/common.cc
----------------------------------------------------------------------
diff --git a/src/utils/common.cc b/src/utils/common.cc
index 3bba398..01ee317 100644
--- a/src/utils/common.cc
+++ b/src/utils/common.cc
@@ -64,7 +64,14 @@ void Debug() {
   while (0 == i)
     sleep(5);
 }
-
+int ArgPos(int argc, char** arglist, const char* arg) {
+  for (int i = 0; i < argc; i++) {
+    if (strcmp(arglist[i], arg) == 0) {
+      return i;
+    }
+  }
+  return -1;
+}
 void  CreateFolder(const std::string name) {
   struct stat buffer;
   if (stat(name.c_str(), &buffer) != 0) {

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/utils/tool.cc
----------------------------------------------------------------------
diff --git a/src/utils/tool.cc b/src/utils/tool.cc
index ebc0022..084ebea 100644
--- a/src/utils/tool.cc
+++ b/src/utils/tool.cc
@@ -1,4 +1,3 @@
-#include <gflags/gflags.h>
 #include <glog/logging.h>
 #include <algorithm>
 #include <fstream>
@@ -8,12 +7,6 @@
 #include "utils/cluster_rt.h"
 #include "utils/common.h"
 
-#ifndef GFLAGS_GFLAGS_H_
-namespace gflags = google;
-#endif  // GFLAGS_GFLAGS_H_
-
-DEFINE_string(confdir, "conf", "Global config dir");
-
 singa::SingaProto global;
 const int SUCCESS = 0;
 const int ARG_ERR = 1;
@@ -54,9 +47,9 @@ int genhost(char* job_conf) {
     nprocs = std::max(nworker_procs, nserver_procs);
 
   // get available host list from global conf
-  std::fstream hostfile(FLAGS_confdir+"/hostfile");
+  std::fstream hostfile("conf/hostfile");
   if (!hostfile.is_open()) {
-    LOG(ERROR) << "Cannot open file: " << FLAGS_confdir+"/hostfile";
+    LOG(ERROR) << "Cannot open file: " << "conf/hostfile";
     return RUN_ERR;
   }
   std::vector<std::string> hosts;
@@ -129,21 +122,20 @@ int cleanup() {
 
 int main(int argc, char **argv) {
   std::string usage = "Usage: singatool <command> <args>\n"
-      " getlogdir        :  show log dir in global config\n"
-      " create           :  generate a unique job id\n"
-      " genhost JOB_CONF :  generate a host list\n"
-      " list             :  list running singa jobs\n"
-      " listall          :  list all singa jobs\n"
-      " view JOB_ID      :  view procs of a singa job\n"
-      " remove JOB_ID    :  remove a job path in zookeeper\n"
-      " removeall        :  remova all job paths in zookeeper\n"
-      " cleanup          :  clean all singa data in zookeeper\n";
-  // set logging level to ERROR and log to STDERR
-  FLAGS_logtostderr = 1;
-  FLAGS_minloglevel = 2;
+      " getlogdir          :  show log dir in global config\n"
+      " create             :  generate a unique job id\n"
+      " genhost <job conf> :  generate a host list\n"
+      " list               :  list running singa jobs\n"
+      " listall            :  list all singa jobs\n"
+      " view <job id>      :  view procs of a singa job\n"
+      " remove <job id>    :  remove a job path in zookeeper\n"
+      " removeall          :  remova all job paths in zookeeper\n"
+      " cleanup            :  clean all singa data in zookeeper\n";
+  // set logging level to ERROR and log to STDERR only
+  google::LogToStderr();
+  google::SetStderrLogging(google::ERROR);
   google::InitGoogleLogging(argv[0]);
-  gflags::ParseCommandLineFlags(&argc, &argv, true);
-  singa::ReadProtoFromTextFile((FLAGS_confdir+"/singa.conf").c_str(), &global);
+  singa::ReadProtoFromTextFile("conf/singa.conf", &global);
 
   // stat code: ARG_ERR for wrong argument, RUN_ERR for runtime error
   int stat = (argc <= 1) ? ARG_ERR : SUCCESS;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/thirdparty/install.sh
----------------------------------------------------------------------
diff --git a/thirdparty/install.sh b/thirdparty/install.sh
index 04321d2..a70b964 100755
--- a/thirdparty/install.sh
+++ b/thirdparty/install.sh
@@ -84,40 +84,6 @@ function install_czmq()
 	return 0;
 }
 
-function install_gflags()
-{
-	if [ ! -e "gflags-2.1.1.tar.gz" ]
-	then
-		wget http://www.comp.nus.edu.sg/~dbsystem/singa/assets/file/thirdparty/gflags-2.1.1.tar.gz;
-	fi
-	rm -rf gflags-2.1.1;
-	tar zxvf gflags-2.1.1.tar.gz && cd gflags-2.1.1;
-	mkdir build && cd build;
-
-	if [ $# == 1 ]
-		then
-			echo "install gflags in $1"
-			cmake .. -DCMAKE_INSTALL_PREFIX=$1 -DCMAKE_CXX_FLAGS=-fPIC;
-			make && make install;
-		elif [ $# == 0 ]
-		then 
-			echo "install gflags in default path";
-			cmake .. -DCMAKE_CXX_FLAGS=-fPIC;
-			make && sudo make install;
-		else
-			echo "gflags is done";
-	fi
-	
-	if [ $? -ne 0 ]
-	then
-		cd ../..;
-		return -1;
-	fi
-	
-	cd ../..;
-	return 0;
-}
-
 function install_glog()
 {
 	if [ ! -e "glog-0.3.3.tar.gz" ]
@@ -448,27 +414,6 @@ do
 			shift
 		fi
 		;;
-	"gflags")
-		echo "install gflags";
-		if [[ $2 == */* ]];then
-			install_gflags $2;
-		    if [ $? -ne 0 ] 
-		    then
-		        echo "ERROR during gflags installation" ;
-		        exit;
-		    fi  
-			shift
-			shift
-		else
-			install_gflags;
-		    if [ $? -ne 0 ] 
-		    then
-		        echo "ERROR during gflags installation" ;
-		        exit;
-		    fi  
-			shift
-		fi
-		;;
 	"glog")
 		echo "install glog";
 		if [[ $2 == */* ]];then
@@ -637,12 +582,6 @@ do
 		        echo "ERROR during czmq installation" ;
 		        exit;
 		    fi  
-			install_gflags $2;
-		    if [ $? -ne 0 ] 
-		    then
-		        echo "ERROR during gflags installation" ;
-		        exit;
-		    fi  
 			install_glog $2;
 		    if [ $? -ne 0 ] 
 		    then
@@ -700,12 +639,6 @@ do
 		        echo "ERROR during czmq installation" ;
 		        exit;
 		    fi  
-			install_gflags;
-		    if [ $? -ne 0 ] 
-		    then
-		        echo "ERROR during gflags installation" ;
-		        exit;
-		    fi  
 			install_glog;
 		    if [ $? -ne 0 ] 
 		    then
@@ -750,7 +683,6 @@ do
 		echo " MISSING_LIBRARY_NAME can be:	"
 		echo "	cmake"
 		echo "	czmq"
-		echo "	gflags"
 		echo "	glog"
 		echo "	lmdb"
 		echo "	OpenBLAS"


Mime
View raw message