singa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wan...@apache.org
Subject [1/3] incubator-singa git commit: SINGA-115 - Print layer debug information in the neural net graph file
Date Fri, 25 Dec 2015 09:14:57 GMT
Repository: incubator-singa
Updated Branches:
  refs/heads/master b0033533d -> 138599fd1


SINGA-115 - Print layer debug information in the neural net graph file

Updated the Graph class to format the string representation in ToJson function.
The Layer::ToString function is updated to generate string for debug info, e.g., layer data
and gradient norm.
Insert code to collect debug info from layers forward and backward function in BPWorker class.
Add NeuralNet::ToGraph function to convert a neualnet into a Graph which can converted into
a string using ToJson function.

The Driver class would generate the node-link representation in a json file in workspace/visualization/train-net.json
file.

After enabling the debug option in job conf, i.e., `debug: true`, there will be two files
for each BP iteration, named fp-stepxx-locxxx.json and
bp-stepxxx-locxxx.json, representing the norm of data/gradient associated of each layer/param.
These files can be used to generate images for visualization via tools/graph.py
We may later consider generating images directly, like Graph::ToImage, using Graphviz library.


Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/1f977b1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/1f977b1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/1f977b1a

Branch: refs/heads/master
Commit: 1f977b1a1701ccf6958ebecb3ad3858da93b3578
Parents: b003353
Author: Wei Wang <wangwei@comp.nus.edu.sg>
Authored: Thu Dec 24 16:44:10 2015 +0800
Committer: Wei Wang <wangwei@comp.nus.edu.sg>
Committed: Thu Dec 24 16:44:10 2015 +0800

----------------------------------------------------------------------
 include/singa/neuralnet/neuralnet.h |   6 ++
 include/singa/utils/common.h        |  11 ++-
 include/singa/utils/graph.h         | 112 +++++++++++++++++++++++------
 src/driver.cc                       |   2 +
 src/neuralnet/layer.cc              |  19 +++--
 src/neuralnet/neuralnet.cc          |  28 +++++++-
 src/utils/common.cc                 |  22 ++++--
 src/utils/graph.cc                  | 117 +++++++++++++++++++++++++++----
 src/worker.cc                       |  23 ++++--
 9 files changed, 278 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/include/singa/neuralnet/neuralnet.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuralnet.h b/include/singa/neuralnet/neuralnet.h
index 7dd4edc..be6fcc5 100644
--- a/include/singa/neuralnet/neuralnet.h
+++ b/include/singa/neuralnet/neuralnet.h
@@ -110,6 +110,12 @@ class NeuralNet {
   }
   inline Param* paramid2param(int id) const { return paramid2param_.at(id); }
 
+  /**
+   * Conver the neural net into graph representation.
+   * Each layer is converted into a node.
+   * @param include_shape if true label the node with shape info
+   */
+  const Graph ToGraph(bool include_shape) const;
  protected:
   /**
    * Create a neural net graph, one node for each layer.

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/include/singa/utils/common.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/common.h b/include/singa/utils/common.h
index ef47b6c..afbe954 100644
--- a/include/singa/utils/common.h
+++ b/include/singa/utils/common.h
@@ -7,9 +7,9 @@
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
-* 
+*
 *   http://www.apache.org/licenses/LICENSE-2.0
-* 
+*
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -32,6 +32,8 @@
 
 namespace singa {
 
+using std::vector;
+using std::string;
 std::string IntVecToString(const std::vector<int>& vec);
 std::string VStringPrintf(std::string fmt, va_list l);
 std::string StringPrintf(std::string fmt, ...);
@@ -149,7 +151,10 @@ void WriteProtoToTextFile(const Message& proto, const char* filename);
 void ReadProtoFromBinaryFile(const char* filename, Message* proto);
 void WriteProtoToBinaryFile(const Message& proto, const char* filename);
 
-
+/**
+ * Write a string (e.g., graph reprensetation of a net) into a text file.
+ */
+void WriteStringToTextFile(const string& filename, const string& context);
 }  // namespace singa
 
 #endif  // SINGA_UTILS_COMMON_H_

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/include/singa/utils/graph.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/graph.h b/include/singa/utils/graph.h
index bad7b19..d26a8ee 100644
--- a/include/singa/utils/graph.h
+++ b/include/singa/utils/graph.h
@@ -7,9 +7,9 @@
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
-* 
+*
 *   http://www.apache.org/licenses/LICENSE-2.0
-* 
+*
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -26,19 +26,36 @@
 #include <string>
 #include <map>
 #include <vector>
-
 namespace singa {
+using std::string;
+using std::map;
 
+/**
+ * Node class representing a layer in a neural net.
+ *
+ * TODO remove layer dependent fields, like origin, and partition_id, to make
+ * it an independent and simple class.
+ */
 class Node {
  public:
   /**
    * Node constructor.
    *
-   * @param name name of the corresponding layer
+   * @param name identifier of the node, e.g, layer name.
    */
-  explicit Node(std::string name);
+  explicit Node(string name);
   /**
-   * Node constructor.
+   * Construct a node with specified attributes.
+   * @param name node identifier
+   * @param attrs node attributes for printing, including "shape", "color", etc.
+   * Depending on the visulization engine, if using graphviz, then the attribute
+   * list is http://www.graphviz.org/content/attrs.
+   */
+  Node(string name, const std::map<string, string>& attrs);
+  /**
+   * @deprecated {to make the Graph class an independent class.}
+   *
+   * Node constructor used for model partitioning.
    *
    * This node is a partition of some node.
    * @param name node name
@@ -46,22 +63,26 @@ class Node {
    * @param id partition id of this node
    * @param proto conf of the corresponding layer
    */
-  Node(const std::string& name, const std::string& origin, int id, void* proto);
+  Node(const string& name, const std::string& origin, int id, void* proto);
   ~Node() {}  // the proto field is deleted outside by other functions
-  void AddDstNode(Node* dstnode);
-  void AddSrcNode(Node* srcnode);
+
+
+  void AddDstNode(Node* dst);
+  void AddSrcNode(Node* src);
   void RemoveDstNode(Node* dst);
   void RemoveSrcNode(Node* src);
 
-  std::string name = "";
+  string name = "";
   //! name of the origin node/layer from which is node is derived
-  std::string origin = "";
+  string origin = "";
   //! partition id
   int partition_id = 0;
   //! proto of the corresponding layer
   void* proto = nullptr;
   std::vector<Node*> srcnodes;
   std::vector<Node*> dstnodes;
+  //!< node attribute including shape, color, etc.
+  std::map<string, string> attrs;
 };
 
 /**
@@ -73,6 +94,7 @@ class Graph {
  public:
   Graph() {}
   ~Graph();
+  const Graph Reverse() const;
   /**
    * @return all nodes of the graph
    */
@@ -83,34 +105,84 @@ class Graph {
    * @param name node name
    * @return return the node of given name
    */
-  inline Node* node(const std::string& name) const {
+  inline Node* node(const string& name) const {
     return name2node_.at(name);
   }
+  /**
+   * Add an exiting node into this graph.
+   */
   void AddNode(Node* node);
-  Node* AddNode(const std::string& name);
+  /**
+   * Creat an node with the given name and add it into the graph.
+   * @return the newly created node.
+   */
+  Node* AddNode(const string& name);
+  /**
+   * Create an node with the given name and attributes.
+   */
+  Node* AddNode(const string& name, const std::map<string, string>& attrs);
+  /**
+   * Add an edge connecting the two given nodes.
+   */
   void AddEdge(Node* srcnode, Node* dstnode);
-  void AddEdge(const std::string& src, const std::string& dst);
+  /**
+   * Add an edge connecting the two nodes with the given name.
+   */
+  void AddEdge(const string& src, const std::string& dst);
+  /**
+   * Add an edge connecting the two given nodes, the edge attributes are also
+   * given.
+   */
+  void AddEdge(Node* srcnode, Node* dstnode,
+      const std::map<string, string>& attrs);
+  /**
+   * Add an edge connecting the two nodes with the given names, the edge
+   * attributes are also given, which are used for printing.
+   * http://www.graphviz.org/content/attrs
+   */
+  void AddEdge(const string& src, const std::string& dst,
+      const std::map<string, string>& attrs);
+
+  /**
+   * Remove the edge connecting the two given nodes.
+   */
   void RemoveEdge(Node* src, Node* dst);
-  void RemoveEdge(const std::string &src, const std::string& dst);
+  /**
+   * Remove the edge connecting two nodes with the given names.
+   */
+  void RemoveEdge(const string &src, const std::string& dst);
   /**
    * Dump the graph into json string which can be used to draw a picture by
-   * graphviz
+   * graphviz.
+   *
+   * It calls ToJson(const std::map<std::string, std::string>& label) with
+   * empty label mapping.
    */
-  std::string ToJson() const;
+  string ToJson() const;
   /**
    * \copybreif ToJson()
    *
-   * @param info info associated with each node
+   * @param label information to be displayed as label for each node
    */
-  std::string ToJson(const std::map<std::string, std::string>& info) const;
+  string ToJson(const map<std::string, std::string>& label) const;
   /**
    * Do topology sort for all nodes of the graph.
    */
   void Sort();
 
  private:
+  /**
+   *
+   * @return the name of the edge connecting src to dst
+   */
+  const string GetEdgeName(const string& src, const string& dst) const {
+    return src + "-->" + dst;
+  }
+
+ private:
   std::vector<Node*> nodes_;
-  std::map<std::string, Node*> name2node_;
+  std::map<string, Node*> name2node_;
+  std::map<string, std::map<string, string>> edge_attrs_;
 };
 
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index ed1b6b4..9389fde 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -201,6 +201,8 @@ void Driver::Train(const JobProto& job_conf) {
   }
 
   NeuralNet* net = NeuralNet::Create(job_conf.neuralnet(), kTrain, grp_size);
+  WriteStringToTextFile(cluster->vis_folder() + "/train_net.json",
+      net->ToGraph(true).ToJson());
   const vector<Worker*> workers = CreateWorkers(job_conf, net);
   const vector<Server*> servers = CreateServers(job_conf, net);
 

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/src/neuralnet/layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc
index 1953d6c..3698b21 100644
--- a/src/neuralnet/layer.cc
+++ b/src/neuralnet/layer.cc
@@ -46,19 +46,18 @@ Layer* Layer::Create(const LayerProto& proto) {
 const std::string Layer::ToString(bool debug, int flag) {
   if (!debug)
     return "";
-  string ret = StringPrintf("Layer %10s ", name().c_str());
+  string ret = "";
   if ((flag & kForward) == kForward && data_.count() !=0) {
-    ret += StringPrintf("data norm1 %13.9f", Asum(data_));
+    ret += StringPrintf("data:%13.9f ", Asum(data_));
+    for (Param* p : GetParams())
+      ret += StringPrintf("%s:%13.9f ",
+          p->name().c_str(), Asum(p->data()));
   }
   if ((flag & kBackward) == kBackward && grad_.count() != 0) {
-      ret += StringPrintf("grad norm1 %13.9f\n", Asum(grad_));
-  }
-  if ((flag & kTrain) == kTrain) {
-    for (Param* p : GetParams()) {
-      ret += StringPrintf(
-          "param id %2d, name %10s, value norm1 %13.9f, grad norm1 %13.9f\n",
-          p->id(), p->name().c_str(), Asum(p->data()), Asum(p->grad()));
-    }
+    ret += StringPrintf("grad:%13.9f ", Asum(grad_));
+    for (Param* p : GetParams())
+      ret += StringPrintf("%s:%13.9f ",
+          p->name().c_str(), Asum(p->grad()));
   }
   return ret;
 }

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index eb8c270..acfc9ea 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -356,7 +356,7 @@ Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions)
{
     }
   }
   graph->Sort();
-  DLOG(INFO) << "Pure graph structure\n" << graph->ToJson();
+  // DLOG(INFO) << "Pure graph structure\n" << graph->ToJson();
   return graph;
 }
 
@@ -383,7 +383,7 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
   for (Node* node : graph->nodes()) {
     auto layer = name2layer(node->name);
     layer->Setup(*(static_cast<LayerProto*>(node->proto)), srclayers(layer));
-    LOG(INFO) << "constructing graph: " << layer->name();
+    DLOG(INFO) << "constructing graph: " << layer->name();
     layerinfo[layer->name()] = IntVecToString(layer->data(nullptr).shape());
     string param_name = "$";
     for (auto param : layer->GetParams()) {
@@ -397,7 +397,7 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
     if (layer->partition_dim() == 0)
       share_param_layers[node->origin].push_back(layer);
   }
-  LOG(INFO) << "Neural net structure\n"  << graph->ToJson(layerinfo);
+  // LOG(INFO) << "Neural net structure\n"  << graph->ToJson(layerinfo);
   // create map from param name to param ptr
   std::unordered_map<string, Param*> name2param;
   for (auto layer : layers_) {
@@ -448,4 +448,26 @@ void NeuralNet::PrepareDataStructures() {
   }
 }
 
+const Graph NeuralNet::ToGraph(bool include_shape) const {
+  Graph g;
+  map<string, string> attrs;
+  attrs["shape"] = "box";
+  vector<string> colors {"black", "red", "yellow", "blue"};
+  for (auto layer : layers_) {
+    LOG_IF(WARNING, layer->partition_id() >= static_cast<int>(colors.size()))
+      << "Too many partitions for displaying";
+    attrs["color"] = colors[layer->partition_id() % colors.size()];
+    if (include_shape) {
+      attrs["label"] = "shape: ";
+      for (const auto& x : layer->data(nullptr).shape())
+        attrs["label"] += std::to_string(x) + " ";
+    }
+    g.AddNode(layer->name(), attrs);
+  }
+
+  for (auto layer : layers_)
+    for (auto src : src_map_.at(layer))
+      g.AddEdge(src->name(), layer->name());
+  return g;
+}
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/src/utils/common.cc
----------------------------------------------------------------------
diff --git a/src/utils/common.cc b/src/utils/common.cc
index de4f906..c75a139 100644
--- a/src/utils/common.cc
+++ b/src/utils/common.cc
@@ -78,6 +78,8 @@
 #include <fcntl.h>
 #include <cfloat>
 
+#include <fstream>
+
 #include <glog/logging.h>
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
@@ -85,8 +87,6 @@
 
 namespace singa {
 
-using std::string;
-using std::vector;
 const int kBufLen = 1024;
 
 string IntVecToString(const vector<int>& vec) {
@@ -142,7 +142,7 @@ const vector<vector<int>> Slice(int num, const vector<int>&
sizes) {
       avg += x;
   avg = avg / num + avg % num;
   int diff = avg / 10;
-  LOG(INFO) << "Slicer, param avg = " << avg << ", diff = " << diff;
+  // DLOG(INFO) << "Slicer, param avg = " << avg << ", diff = " <<
diff;
 
   int capacity = avg, nbox = 0;
   for (int x : sizes) {
@@ -172,7 +172,7 @@ const vector<vector<int>> Slice(int num, const vector<int>&
sizes) {
         slicestr += ", " + std::to_string(size);
       }
     }
-    LOG(INFO) << slicestr;
+    // DLOG(INFO) << slicestr;
     slices.push_back(slice);
   }
   CHECK_LE(nbox, num);
@@ -213,8 +213,7 @@ const vector<int> PartitionSlices(int num, const vector<int>&
slices) {
     }
     disp += " " + std::to_string(slices[i]);
   }
-  LOG(INFO) << "partition slice (avg = " << avg
-            << ", num = " << num << "):" << disp;
+  // DLOG(INFO) << "partition slice (avg = " << avg << ", num = " <<
num << "):" << disp;
   return slice2box;
 }
 
@@ -561,4 +560,15 @@ void WriteProtoToBinaryFile(const Message& proto, const char* filename)
{
   CHECK_NE(fd, -1) << "File cannot open: " << filename;
   CHECK(proto.SerializeToFileDescriptor(fd));
 }
+
+
+
+void WriteStringToTextFile(const string& filename, const string& context) {
+  std::ofstream ofs;
+  ofs.open(filename);
+  CHECK(ofs.is_open()) << "Can't write to file: " << filename;
+  ofs << context;
+  ofs.flush();
+  ofs.close();
+}
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/src/utils/graph.cc
----------------------------------------------------------------------
diff --git a/src/utils/graph.cc b/src/utils/graph.cc
index 6b3091e..9a23773 100644
--- a/src/utils/graph.cc
+++ b/src/utils/graph.cc
@@ -7,9 +7,9 @@
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
-* 
+*
 *   http://www.apache.org/licenses/LICENSE-2.0
-* 
+*
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,6 +20,7 @@
 *************************************************************/
 
 #include "singa/utils/graph.h"
+#include "singa/utils/common.h"
 
 #include <glog/logging.h>
 #include <algorithm>
@@ -32,9 +33,16 @@ using std::map;
 using std::string;
 using std::vector;
 
+/**************************************************************************
+ * Implementation for Node class
+ *************************************************************************/
 Node::Node(string name) {
   this->name = name;
 }
+Node::Node(string name, const std::map<string, string>& attrs) {
+  this->name = name;
+  this->attrs = attrs;
+}
 
 Node::Node(const string& name, const string& origin, int id, void* proto) {
   this->name = name;
@@ -67,6 +75,10 @@ void Node::RemoveSrcNode(Node* src) {
   srcnodes.erase(iter);
 }
 
+/****************************************************************************
+ * Implementation for Graph class
+ ****************************************************************************/
+
 Graph::~Graph() {
   for (Node* node : nodes_)
     delete node;
@@ -85,6 +97,13 @@ Node* Graph::AddNode(const string& name) {
   return node;
 }
 
+Node* Graph::AddNode(const string& name, const std::map<string, string>& attrs)
+{
+  Node* node = new Node(name, attrs);
+  AddNode(node);
+  return node;
+}
+
 void Graph::AddEdge(Node* srcnode, Node* dstnode) {
   srcnode->AddDstNode(dstnode);
   dstnode->AddSrcNode(srcnode);
@@ -97,6 +116,16 @@ void Graph::AddEdge(const string& src, const string& dst) {
   CHECK(dstnode != name2node_.end()) << "can't find dst node " << dst;
   AddEdge(srcnode->second, dstnode->second);
 }
+void Graph::AddEdge(Node* srcnode, Node* dstnode,
+      const std::map<string, string>& attrs) {
+  AddEdge(srcnode, dstnode);
+  edge_attrs_[GetEdgeName(srcnode->name, dstnode->name)] = attrs;
+}
+void Graph::AddEdge(const string& src, const std::string& dst,
+      const std::map<string, string>& attrs) {
+  AddEdge(src, dst);
+  edge_attrs_[GetEdgeName(src, dst)] = attrs;
+}
 
 void Graph::RemoveEdge(Node* src, Node* dst) {
   src->RemoveDstNode(dst);
@@ -112,10 +141,11 @@ void Graph::RemoveEdge(const string &src, const string& dst)
{
 }
 
 string Graph::ToJson() const {
-  map<string, string> info;
-  return ToJson(info);
+  map<string, string> label;
+  return ToJson(label);
 }
 
+/*
 string Graph::ToJson(const map<string, string>& info) const {
   map<string, int> nodeid;
   string disp = "{\"directed\":1,\n";
@@ -126,7 +156,7 @@ string Graph::ToJson(const map<string, string>& info) const
{
   vector<string> colors = {"red", "blue", "black", "green"};
 
   // see for more shapes at http://www.graphviz.org/doc/info/shapes.html
-  vector<string> shapes = {"box", "ellipse"};
+  // vector<string> shapes = {"box", "ellipse"};
   int id = 0;
   for (auto node : nodes_) {
     char str[1024];
@@ -134,14 +164,10 @@ string Graph::ToJson(const map<string, string>& info) const
{
     string color = colors[(node->partition_id)%colors.size()];
     string shape;
     string origin = node->origin;
-    if (origin.find("##") != string::npos)
-      shape = shapes[1];
-    else
-      shape = shapes[0];
     snprintf(str, sizeof(str),
         "{\"id\":\"%s%s\", \"color\":\"%s\",\"shape\":\"%s\"}\n", name.c_str(),
         info.find(name) != info.end() ? info.at(name).c_str() : "",
-        color.c_str(), shape.c_str());
+        color.c_str(), "box");
     if (!first)
       disp += ",";
     else
@@ -170,6 +196,7 @@ string Graph::ToJson(const map<string, string>& info) const
{
   disp += "]\n";
   return disp+"}";
 }
+*/
 
 // sort to make `bottom' nodes be placed in the front positions
 void Graph::Sort() {
@@ -227,11 +254,73 @@ void Graph::Sort() {
       visiting_nodes.push(node);
     }
   }
-  for (auto node : nodes_) {
-    LOG(INFO) << "nodes: " << node->name;
-  }
-  LOG(INFO) << "finish printing nodes ";
   CHECK_EQ(nodes_.size(), n);
 }
 
+const Graph Graph::Reverse() const {
+  Graph g;
+  for (Node* n : nodes_)
+    g.AddNode(n->name, n->attrs);
+  for (Node* src : nodes_)
+    for (Node* dst : src->dstnodes) {
+      map<string, string> attrs;
+      const string edge = GetEdgeName(src->name, dst->name);
+      if (edge_attrs_.find(edge) != edge_attrs_.end())
+        attrs = edge_attrs_.at(edge);
+      g.AddEdge(dst->name, src->name, attrs);
+    }
+  return g;
+}
+
+string Graph::ToJson(const map<string, string>& label) const {
+  string disp = "{\"directed\":1,\n";
+
+  // add nodes
+  disp += "\"nodes\":[\n";
+
+  bool first = true;
+  map<string, int> node_id;
+  int id = 0;
+  for (auto node : nodes_) {
+    string name = node->name;
+    string lbl = name + " -- ";
+    if (label.find(name) != label.end())
+      lbl += label.at(name);
+    if (node->attrs.find("label") != node->attrs.end())
+      lbl += node->attrs.at("label");
+    disp += StringPrintf("%c{\"id\":\"%s\", \"label\":\"%s\"",
+        !first ? ',' : ' ', name.c_str(), lbl.c_str());
+    for (const auto& attr : node->attrs)
+      if (attr.first != "label")
+        disp += StringPrintf(", \"%s\":\"%s\"",
+            attr.first.c_str(), attr.second.c_str());
+    disp += "}\n";
+    first = false;
+    node_id[name] = id++;
+  }
+  disp += "]\n,\n";
+
+  // add edges
+  disp += "\"links\":[\n";
+  first = true;
+  for (auto src : nodes_) {
+    for (auto dst : src->dstnodes) {
+      const string edge_name = GetEdgeName(src->name, dst->name);
+      string lbl = "";
+      if (label.find(edge_name) != label.end())
+        lbl = label.at(edge_name);
+      disp += StringPrintf("%c{\"source\":%d, \"target\":%d, \"label\": \"%s\"",
+          !first ? ',' : ' ', node_id[src->name], node_id[dst->name],
+          lbl.c_str());
+      if (edge_attrs_.find(edge_name) != edge_attrs_.end()) {
+        for (const auto& attr : edge_attrs_.at(edge_name))
+          disp += StringPrintf(", \"%s\":\"%s\"",
+              attr.first.c_str(), attr.second.c_str());
+      }
+      disp += "}\n";
+      first = false;
+    }
+  }
+  return disp + "]}";
+}
 }  // namespace singa

http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/1f977b1a/src/worker.cc
----------------------------------------------------------------------
diff --git a/src/worker.cc b/src/worker.cc
index 01872d0..23868da 100644
--- a/src/worker.cc
+++ b/src/worker.cc
@@ -320,12 +320,6 @@ void Worker::Display(int flag, const std::string& prefix, NeuralNet*
net) {
       const string& disp = layer->ToString(false, flag);
       if (disp.length())
         LOG(ERROR) << prefix << "  " << disp;
-      if (job_conf_.debug()) {
-        const string& info = layer->ToString(true, flag);
-        if (info.length()) {
-          LOG(INFO) << prefix << info;
-        }
-      }
     }
   }
 }
@@ -341,6 +335,7 @@ void BPWorker::TestOneBatch(int step, Phase phase, NeuralNet* net) {
 }
 
 void BPWorker::Forward(int step, Phase phase, NeuralNet* net) {
+  map<string, string> label;
   for (auto& layer : net->layers()) {
     if (layer->partition_id() == id_) {
       // TODO(wangwei): enable this for model partition
@@ -355,15 +350,24 @@ void BPWorker::Forward(int step, Phase phase, NeuralNet* net) {
       }
       // LOG(ERROR) << layer->name() << " forward";
       layer->ComputeFeature(phase | kForward, net->srclayers(layer));
+      if (job_conf_.debug() && grp_id_ == 0)
+        label[layer->name()] = layer->ToString(true, phase | kForward);
+
       // TODO(wangwei): enable this for model partition
       // send data to other workers
       // if (typeid(*layer) == typeid(BridgeSrcLayer))
       //   SendBlobs(true, false, dynamic_cast<BridgeLayer*>(layer), net);
     }
   }
+  if (label.size()) {
+    const string path = Cluster::Get()->vis_folder() + "/fp-step"
+      + std::to_string(step) +"-loc" + std::to_string(id_) + ".json";
+    WriteStringToTextFile(path, net->ToGraph(false).ToJson(label));
+  }
 }
 
 void BPWorker::Backward(int step, NeuralNet* net) {
+  map<string, string> label;
   auto& layers = net->layers();
   for (auto it = layers.rbegin(); it != layers.rend(); it++) {
     Layer* layer = *it;
@@ -374,6 +378,8 @@ void BPWorker::Backward(int step, NeuralNet* net) {
       //   ReceiveBlobs(false, true, layer, net);
       // LOG(ERROR) << layer->name() << " backward";
       layer->ComputeGradient(kTrain | kBackward, net->srclayers(layer));
+      if (job_conf_.debug() && grp_id_ == 0)
+        label[layer->name()] = layer->ToString(true, kTrain | kBackward);
       for (Param* p : layer->GetParams())
         Update(step, p);
       // TODO(wangwei): enable this for model partition
@@ -382,6 +388,11 @@ void BPWorker::Backward(int step, NeuralNet* net) {
       //   SendBlobs(false, true, dynamic_cast<BridgeDstLayer*>(layer), net);
     }
   }
+  if (label.size()) {
+    const string path = Cluster::Get()->vis_folder() + "/bp-step"
+      + std::to_string(step) + "-loc" + std::to_string(id_) + ".json";
+    WriteStringToTextFile(path, net->ToGraph(false).Reverse().ToJson(label));
+  }
 }
 
 /****************************CDWorker**********************************/


Mime
View raw message