giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From clau...@apache.org
Subject git commit: updated refs/heads/trunk to f8a3c77
Date Tue, 25 Jun 2013 22:00:07 GMT
Updated Branches:
  refs/heads/trunk 7ea62d9c1 -> f8a3c777e


GIRAPH-696: Should be able to spill giraph metrics to a specified directory on HDFS


Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/f8a3c777
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/f8a3c777
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/f8a3c777

Branch: refs/heads/trunk
Commit: f8a3c777e1fc1902a6a965c1d2a33e7ce3cc1643
Parents: 7ea62d9
Author: Claudio Martella <claudio@apache.org>
Authored: Tue Jun 25 23:59:32 2013 +0200
Committer: Claudio Martella <claudio@apache.org>
Committed: Tue Jun 25 23:59:32 2013 +0200

----------------------------------------------------------------------
 CHANGELOG                                       |  3 ++
 .../org/apache/giraph/conf/GiraphConstants.java |  5 +++
 .../apache/giraph/master/BspServiceMaster.java  | 41 +++++++++++++++++++-
 3 files changed, 48 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 36693ab..0f2758e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
 Giraph Change Log
 
 Release 1.1.0 - unreleased
+  GIRAPH-696: Should be able to spill giraph metrics to a specified 
+  directory on HDFS (claudio)
+
   GIRAPH-695: Add getCombiner and getComputation methods for 
   MasterCompute (aching)
 

http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
index 80afab9..c65b5f0 100644
--- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
+++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java
@@ -315,6 +315,11 @@ public interface GiraphConstants {
       new BooleanConfOption("giraph.metrics.enable", false,
           "Enable the Metrics system");
 
+  /** Directory in HDFS to write master metrics to, instead of stderr */
+  StrConfOption METRICS_DIRECTORY =
+      new StrConfOption("giraph.metrics.directory", "",
+          "Directory in HDFS to write master metrics to, instead of stderr");
+
   /**
    *  ZooKeeper comma-separated list (if not set,
    *  will start up ZooKeeper locally)

http://git-wip-us.apache.org/repos/asf/giraph/blob/f8a3c777/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java b/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
index 0d266a6..3558887 100644
--- a/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
+++ b/giraph-core/src/main/java/org/apache/giraph/master/BspServiceMaster.java
@@ -91,6 +91,7 @@ import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -974,7 +975,11 @@ public class BspServiceMaster<I extends WritableComparable,
     }
 
     if (conf.metricsEnabled()) {
-      aggregatedMetrics.print(superstep, System.err);
+      if (GiraphConstants.METRICS_DIRECTORY.isDefaultValue(conf)) {
+        aggregatedMetrics.print(superstep, System.err);
+      } else {
+        printAggregatedMetricsToHDFS(superstep, aggregatedMetrics);
+      }
     }
 
     if (LOG.isInfoEnabled()) {
@@ -985,6 +990,40 @@ public class BspServiceMaster<I extends WritableComparable,
   }
 
   /**
+   * Write superstep metrics to own file in HDFS
+   * @param superstep the current superstep
+   * @param aggregatedMetrics the aggregated metrics to write
+   */
+  private void printAggregatedMetricsToHDFS(
+      long superstep, AggregatedMetrics aggregatedMetrics) {
+    ImmutableClassesGiraphConfiguration conf = getConfiguration();
+    PrintStream out = null;
+    Path dir = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf));
+    Path outFile = new Path(GiraphConstants.METRICS_DIRECTORY.get(conf) +
+        Path.SEPARATOR_CHAR + "superstep_" + superstep + ".metrics");
+    try {
+      FileSystem fs;
+      fs = FileSystem.get(conf);
+      if (!fs.exists(dir)) {
+        fs.mkdirs(dir);
+      }
+      if (fs.exists(outFile)) {
+        throw new RuntimeException(
+            "printAggregatedMetricsToHDFS: metrics file exists");
+      }
+      out = new PrintStream(fs.create(outFile));
+      aggregatedMetrics.print(superstep, out);
+    } catch (IOException e) {
+      throw new RuntimeException(
+          "printAggregatedMetricsToHDFS: error creating metrics file", e);
+    } finally {
+      if (out != null) {
+        out.close();
+      }
+    }
+  }
+
+  /**
    * Finalize the checkpoint file prefixes by taking the chosen workers and
    * writing them to a finalized file.  Also write out the master
    * aggregated aggregator array from the previous superstep.


Mime
View raw message