singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [1/2] incubator-singa git commit: SINGA-25 - Setup glog output path
Date Sat, 27 Jun 2015 07:14:13 GMT
Repository: incubator-singa
Updated Branches:
  refs/heads/master 14ce5d9ae -> f234c4972


SINGA-25 - Setup glog output path

Set the output files of glog to workspace/glog.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/90c26945
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/90c26945
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/90c26945

Branch: refs/heads/master
Commit: 90c2694539d5fc3119f59d4dfc28db1954b04307
Parents: 14ce5d9
Author: wang wei <wangwei@comp.nus.edu.sg>
Authored: Fri Jun 26 22:46:58 2015 +0800
Committer: wang wei <wangwei@comp.nus.edu.sg>
Committed: Fri Jun 26 22:46:58 2015 +0800

----------------------------------------------------------------------
 include/utils/common.h | 16 +++++++++----
 src/main.cc            | 11 +++++----
 src/trainer/server.cc  |  6 +++--
 src/trainer/worker.cc  |  6 +++--
 src/utils/common.cc    | 57 +++++++++++++++++++++++++++++++--------------
 5 files changed, 66 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/90c26945/include/utils/common.h
----------------------------------------------------------------------
diff --git a/include/utils/common.h b/include/utils/common.h
index d3c23a8..619e06a 100644
--- a/include/utils/common.h
+++ b/include/utils/common.h
@@ -2,6 +2,7 @@
 #define SINGA_UTILS_COMMON_H_
 
 #include <google/protobuf/message.h>
+#include <stdlib.h>
 #include <map>
 #include <sstream>
 #include <string>
@@ -9,9 +10,9 @@
 
 namespace singa {
 
-std::string IntVecToString(const std::vector<int>& vec) ;
-std::string VStringPrintf(std::string fmt, va_list l) ;
-std::string StringPrintf(std::string fmt, ...) ;
+std::string IntVecToString(const std::vector<int>& vec);
+std::string VStringPrintf(std::string fmt, va_list l);
+std::string StringPrintf(std::string fmt, ...);
 void ReadProtoFromTextFile(const char* filename,
                            google::protobuf::Message* proto);
 void WriteProtoToTextFile(const google::protobuf::Message& proto,
@@ -21,6 +22,8 @@ void ReadProtoFromBinaryFile(const char* filename,
 void WriteProtoToBinaryFile(const google::protobuf::Message& proto,
                             const char* filename);
 
+const std::string CurrentDateTime();
+void  CreateFolder(const std::string name);
 /*
 inline void Sleep(int millisec=1){
   std::this_thread::sleep_for(std::chrono::milliseconds(millisec));
@@ -29,10 +32,13 @@ inline void Sleep(int millisec=1){
 
 int gcd(int a, int b);
 int LeastCommonMultiple(int a, int b);
-inline float rand_real(){
-  return  static_cast<float>(rand())/(RAND_MAX+1.0f);
+/*
+inline float rand_real() {
+  return  static_cast<float>(rand_r())/(RAND_MAX+1.0f);
 }
+*/
 const std::string GetHostIP();
+void SetupLog(const std::string& workspace, const std::string& model);
 
 class Metric {
  public:

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/90c26945/src/main.cc
----------------------------------------------------------------------
diff --git a/src/main.cc b/src/main.cc
index 851d528..2a52c07 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -1,6 +1,7 @@
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 #include "trainer/trainer.h"
+#include "utils/common.h"
 #ifndef GFLAGS_GFLAGS_H_
   namespace gflags = google;
 #endif  // GFLAGS_GFLAGS_H_
@@ -29,11 +30,11 @@ DEFINE_string(model, "examples/mnist/conv.conf", "Model config file");
  * If users want to use their own implemented classes, they should register
  * them here. Refer to the Worker::RegisterDefaultClasses()
  */
-void RegisterClasses(const singa::ModelProto& proto){
+void RegisterClasses(const singa::ModelProto& proto) {
 }
 
+
 int main(int argc, char **argv) {
-  // TODO set log dir
   google::InitGoogleLogging(argv[0]);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
@@ -41,8 +42,10 @@ int main(int argc, char **argv) {
   singa::ReadProtoFromTextFile(FLAGS_cluster.c_str(), &cluster);
   singa::ModelProto model;
   singa::ReadProtoFromTextFile(FLAGS_model.c_str(), &model);
-  LOG(INFO)<<"The cluster config is\n"<<cluster.DebugString();
-  LOG(INFO)<<"The model config is\n"<<model.DebugString();
+  singa::SetupLog(cluster.workspace(), model.name());
+
+  LOG(INFO) << "The cluster config is\n" << cluster.DebugString();
+  LOG(INFO) << "The model config is\n" << model.DebugString();
 
   RegisterClasses(model);
   singa::Trainer trainer;

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/90c26945/src/trainer/server.cc
----------------------------------------------------------------------
diff --git a/src/trainer/server.cc b/src/trainer/server.cc
index 21ff21a..42d6a79 100644
--- a/src/trainer/server.cc
+++ b/src/trainer/server.cc
@@ -25,7 +25,8 @@ void Server::Setup(const UpdaterProto& proto,
 }
 
 void Server::Run(){
-  LOG(ERROR)<<"Server (group_id= "<<group_id_<<", id="<<server_id_<<")
starts";
+  LOG(ERROR)<<"Server (group_id = "<<group_id_
+    <<", id = "<<server_id_<<") starts";
   dealer_=std::make_shared<Dealer>(2*thread_id_);
   dealer_->Connect(kInprocRouterEndpoint);
   auto cluster=Cluster::Get();
@@ -113,7 +114,8 @@ void Server::Run(){
     if (response!=nullptr)
       dealer_->Send(&response);
   }
-  LOG(INFO)<<"Server (group_id= "<<group_id_<<", id="<<server_id_<<")
stops";
+  LOG(ERROR)<<"Server (group_id = "<<group_id_
+    <<", id = "<<server_id_<<") stops";
 }
 
 Msg* Server::HandlePut(Msg **msg){

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/90c26945/src/trainer/worker.cc
----------------------------------------------------------------------
diff --git a/src/trainer/worker.cc b/src/trainer/worker.cc
index 5d301dc..a92ba2c 100644
--- a/src/trainer/worker.cc
+++ b/src/trainer/worker.cc
@@ -47,7 +47,8 @@ void Worker::ConnectStub(shared_ptr<Dealer> dealer, EntityType type){
 }
 
 void Worker::Run(){
-  LOG(ERROR)<<"Worker (group_id= "<<group_id_<<", id="<<worker_id_<<")
starts";
+  LOG(ERROR)<<"Worker (group_id = "<<group_id_
+    <<", id = "<<worker_id_<<") starts";
   dealer_=make_shared<Dealer>(2*thread_id_);
   ConnectStub(dealer_, kWorkerParam);
   for(auto layer: train_net_->layers())
@@ -90,7 +91,8 @@ void Worker::Run(){
   }
 
   Stop();
-  LOG(INFO)<<"Worker (group_id= "<<group_id_<<", id="<<worker_id_<<")
stops";
+  LOG(ERROR)<<"Worker (group_id = "<<group_id_
+    <<", id = "<<worker_id_<<") stops";
 }
 
 void Worker::Stop(){

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/90c26945/src/utils/common.cc
----------------------------------------------------------------------
diff --git a/src/utils/common.cc b/src/utils/common.cc
index a3242aa..1f3f1b6 100644
--- a/src/utils/common.cc
+++ b/src/utils/common.cc
@@ -8,14 +8,15 @@
 #include <stdarg.h>
 #include <stdio.h>
 #include <unistd.h>
+#include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/ioctl.h>
 #include <netinet/in.h>
 #include <net/if.h>
 #include <arpa/inet.h>
-
-
+#include <time.h>
+#include <string>
 
 namespace singa {
 
@@ -28,18 +29,18 @@ using google::protobuf::io::ZeroCopyInputStream;
 using google::protobuf::Message;
 
 const int kBufLen = 1024;
-std::string IntVecToString(const vector<int>& vec) {
-  string disp="(";
-  for(int x: vec)
-    disp+=std::to_string(x)+", ";
-  return disp+")";
+string IntVecToString(const vector<int>& vec) {
+  string disp = "(";
+  for (int x : vec)
+    disp += std::to_string(x) + ", ";
+  return disp + ")";
 }
 /**
  *  * Formatted string.
  *   */
 string VStringPrintf(string fmt, va_list l) {
   char buffer[32768];
-  vsnprintf(buffer, 32768, fmt.c_str(), l);
+  vsnprintf(buffer, sizeof(buffer), fmt.c_str(), l);
   return string(buffer);
 }
 
@@ -48,7 +49,7 @@ string VStringPrintf(string fmt, va_list l) {
  *   */
 string StringPrintf(string fmt, ...) {
   va_list l;
-  va_start(l, fmt); //fmt.AsString().c_str());
+  va_start(l, fmt);  // fmt.AsString().c_str());
   string result = VStringPrintf(fmt, l);
   va_end(l);
   return result;
@@ -64,6 +65,14 @@ void Debug() {
     sleep(5);
 }
 
+void  CreateFolder(const std::string name) {
+  struct stat buffer;
+  if (stat(name.c_str(), &buffer) != 0) {
+    mkdir(name.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+    CHECK_EQ(stat(name.c_str(), &buffer), 0);
+  }
+}
+
 // the proto related functions are from Caffe.
 void ReadProtoFromTextFile(const char* filename, Message* proto) {
   int fd = open(filename, O_RDONLY);
@@ -100,24 +109,22 @@ void WriteProtoToBinaryFile(const Message& proto, const char* filename)
{
   CHECK_NE(fd, -1) << "File cannot open: " << filename;
   CHECK(proto.SerializeToFileDescriptor(fd));
 }
-int gcd(int a, int b)
-{
-  for (;;)
-  {
+
+int gcd(int a, int b) {
+  for (;;) {
     if (a == 0) return b;
     b %= a;
     if (b == 0) return a;
     a %= b;
   }
 }
-int LeastCommonMultiple(int a, int b)
-{
+int LeastCommonMultiple(int a, int b) {
   int temp = gcd(a, b);
 
   return temp ? (a / temp * b) : 0;
 }
 
-const std::string GetHostIP(){
+const string GetHostIP() {
   int fd;
   struct ifreq ifr;
 
@@ -135,7 +142,23 @@ const std::string GetHostIP(){
 
   string ip(inet_ntoa(((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr));
   /* display result */
-  LOG(INFO)<<"Host IP=("<<ip;
+  LOG(INFO) << "Host IP=(" << ip;
   return ip;
 }
+
+void SetupLog(const std::string& workspace, const std::string& model) {
+  std::string folder = workspace+"/log/";
+  // TODO check if NFS, then create folder using script otherwise may have
+  // problems due to multiple processes create the same folder.
+  CreateFolder(folder);
+  std::string warn = folder + model + "-warn-";
+  std::string info = folder + model + "-info-";
+  std::string error = folder + model + "-error-";
+  std::string fatal = folder + model + "-fatal-";
+  google::SetLogDestination(google::WARNING, warn.c_str());
+  google::SetLogDestination(google::INFO, info.c_str());
+  google::SetLogDestination(google::ERROR, error.c_str());
+  google::SetLogDestination(google::FATAL, fatal.c_str());
+}
+
 }  // namespace singa


Mime
View raw message