giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hesl...@apache.org
Subject git commit: updated refs/heads/trunk to 8205f0d
Date Tue, 30 Aug 2016 18:39:42 GMT
Repository: giraph
Updated Branches:
  refs/heads/trunk a15675cae -> 8205f0d26


Out-of-core is logging too aggressively

Summary:
Example aggressive logging at INFO level:

INFO    <datestamp> [ooc-io-0] org.apache.giraph.ooc.policy.ThresholdBasedOracle  -
getNextIOActions: usedMemoryFraction = 0.79
INFO    <datestamp> [ooc-io-0] org.apache.giraph.ooc.OutOfCoreIOCallable  - call: thread
0's next IO command is: LoadPartitionIOCommand: (partitionId = 4676, superstep = 0)
INFO    <datestamp> [ooc-io-0] org.apache.giraph.ooc.OutOfCoreIOCallable  - call: thread
0's command LoadPartitionIOCommand: (partitionId = 4676, superstep = 0) completed: bytes=
0, duration=0, bandwidth=NaN, bandwidth (excluding GC time)=NaN

Test Plan: mvn clean verify -P hadoop_facebook

Reviewers: majakabiljo, maja.kabiljo, sergey.edunov, heslami

Reviewed By: heslami

Subscribers: dionysis.logothetis

Differential Revision: https://reviews.facebook.net/D62853


Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/8205f0d2
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/8205f0d2
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/8205f0d2

Branch: refs/heads/trunk
Commit: 8205f0d26abda68a740de8c962a32172d2bc7e8b
Parents: a15675c
Author: Tyler Serdar Bulut <sbulut@fb.com>
Authored: Tue Aug 30 13:31:13 2016 -0500
Committer: Hassan Eslami <hassan@wirelessprv-10-193-225-240.near.illinois.edu>
Committed: Tue Aug 30 13:39:16 2016 -0500

----------------------------------------------------------------------
 .../main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java | 8 ++++----
 .../org/apache/giraph/ooc/policy/FixedPartitionsOracle.java  | 6 ++++--
 .../org/apache/giraph/ooc/policy/ThresholdBasedOracle.java   | 4 ++--
 3 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/giraph/blob/8205f0d2/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java
index c21be95..d75fee2 100644
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/OutOfCoreIOCallable.java
@@ -76,8 +76,8 @@ public class OutOfCoreIOCallable implements Callable<Void>,
     while (true) {
       oocEngine.getSuperstepLock().readLock().lock();
       IOCommand command = oocEngine.getIOScheduler().getNextIOCommand(diskId);
-      if (LOG.isInfoEnabled() && !(command instanceof WaitIOCommand)) {
-        LOG.info("call: thread " + diskId + "'s next IO command is: " +
+      if (LOG.isDebugEnabled() && !(command instanceof WaitIOCommand)) {
+        LOG.debug("call: thread " + diskId + "'s next IO command is: " +
             command);
       }
       if (command == null) {
@@ -101,8 +101,8 @@ public class OutOfCoreIOCallable implements Callable<Void>,
         timeInGC = oocEngine.getServiceWorker().getGraphTaskManager()
             .getSuperstepGCTime() - timeInGC;
         bytes = command.bytesTransferred();
-        if (LOG.isInfoEnabled() && !(command instanceof WaitIOCommand)) {
-          LOG.info("call: thread " + diskId + "'s command " + command +
+        if (LOG.isDebugEnabled() && !(command instanceof WaitIOCommand)) {
+          LOG.debug("call: thread " + diskId + "'s command " + command +
               " completed: bytes= " + bytes + ", duration=" + duration + ", " +
               "bandwidth=" + String.format("%.2f", (double) bytes / duration *
               1000 / 1024 / 1024) +

http://git-wip-us.apache.org/repos/asf/giraph/blob/8205f0d2/giraph-core/src/main/java/org/apache/giraph/ooc/policy/FixedPartitionsOracle.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/policy/FixedPartitionsOracle.java
b/giraph-core/src/main/java/org/apache/giraph/ooc/policy/FixedPartitionsOracle.java
index 002dc85..da21973 100644
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/policy/FixedPartitionsOracle.java
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/policy/FixedPartitionsOracle.java
@@ -95,8 +95,10 @@ public class FixedPartitionsOracle implements OutOfCoreOracle {
         IOAction.LOAD_PARTITION,
         IOAction.STORE_MESSAGES_AND_BUFFERS};
     } else if (numPartitions > maxPartitionsInMemory) {
-      LOG.warn("getNextIOActions: number of partitions in memory passed the " +
-          "specified threshold!");
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("getNextIOActions: number of partitions in memory passed " +
+          "the specified threshold!");
+      }
       return new IOAction[]{
         IOAction.STORE_PARTITION,
         IOAction.STORE_MESSAGES_AND_BUFFERS};

http://git-wip-us.apache.org/repos/asf/giraph/blob/8205f0d2/giraph-core/src/main/java/org/apache/giraph/ooc/policy/ThresholdBasedOracle.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/ooc/policy/ThresholdBasedOracle.java
b/giraph-core/src/main/java/org/apache/giraph/ooc/policy/ThresholdBasedOracle.java
index ff2b3f7..00a8011 100644
--- a/giraph-core/src/main/java/org/apache/giraph/ooc/policy/ThresholdBasedOracle.java
+++ b/giraph-core/src/main/java/org/apache/giraph/ooc/policy/ThresholdBasedOracle.java
@@ -285,8 +285,8 @@ public class ThresholdBasedOracle implements OutOfCoreOracle {
   @Override
   public IOAction[] getNextIOActions() {
     double usedMemoryFraction = 1 - MemoryUtils.freeMemoryFraction();
-    if (LOG.isInfoEnabled()) {
-      LOG.info(String.format("getNextIOActions: usedMemoryFraction = %.2f",
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(String.format("getNextIOActions: usedMemoryFraction = %.2f",
           usedMemoryFraction));
     }
     if (usedMemoryFraction > highMemoryPressure) {


Mime
View raw message