tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject tez git commit: TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option (rbalamohan)
Date Sun, 16 Aug 2015 23:00:42 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.6 6360867cb -> 86309f93d


TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option (rbalamohan)

(cherry picked from commit a405ce95762ffc314269a0428e260b966a1627da)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/86309f93
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/86309f93
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/86309f93

Branch: refs/heads/branch-0.6
Commit: 86309f93d873f9113f02827ff8ebf472772d3909
Parents: 6360867
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Mon Aug 17 04:23:21 2015 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Mon Aug 17 04:28:48 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                              |  1 +
 .../tez/runtime/library/common/shuffle/FetchResult.java  | 11 +++++++++++
 .../tez/runtime/library/common/shuffle/Fetcher.java      |  6 +-----
 .../tez/runtime/library/common/shuffle/InputHost.java    |  9 +++++++++
 .../library/common/shuffle/impl/ShuffleManager.java      |  3 ++-
 5 files changed, 24 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/86309f93/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 21fa096..4808e5b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ Release 0.6.3: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option
   TEZ-2630. TezChild receives IP address instead of FQDN.
 
 Release 0.6.2: Unreleased

http://git-wip-us.apache.org/repos/asf/tez/blob/86309f93/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
index 1c39a24..d9595f0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
@@ -43,13 +43,20 @@ public class FetchResult {
   private final int port;
   private final int partition;
   private final Iterable<InputAttemptIdentifier> pendingInputs;
+  private final String additionalInfo;
 
   public FetchResult(String host, int port, int partition,
       Iterable<InputAttemptIdentifier> pendingInputs) {
+    this(host, port, partition, pendingInputs, null);
+  }
+
+  public FetchResult(String host, int port, int partition,
+      Iterable<InputAttemptIdentifier> pendingInputs, String additionalInfo) {
     this.host = host;
     this.port = port;
     this.partition = partition;
     this.pendingInputs = pendingInputs;
+    this.additionalInfo = additionalInfo;
   }
 
   public String getHost() {
@@ -67,4 +74,8 @@ public class FetchResult {
   public Iterable<InputAttemptIdentifier> getPendingInputs() {
     return pendingInputs;
   }
+
+  public String getAdditionalInfo() {
+    return additionalInfo;
+  }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/86309f93/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
index 6875966..e4cf071 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
@@ -208,8 +208,6 @@ public class Fetcher implements Callable<FetchResult> {
       if (!multiplex) {
         throw new IOException("server didn't return all expected map outputs: "
             + remaining.size() + " left.");
-      } else {
-        LOG.info("Shared fetch failed to return " + remaining.size() + " inputs on this try");
       }
     }
 
@@ -355,10 +353,8 @@ public class Fetcher implements Callable<FetchResult> {
       lock = getLock();
       if (lock == null) {
         // re-queue until we get a lock
-        LOG.info("Requeuing " + host + ":" + port
-            + " downloads because we didn't get a lock");
         return new HostFetchResult(new FetchResult(host, port, partition,
-            remaining), null, false);
+            remaining, "Requeuing as we didn't get a lock"), null, false);
       } else {
         if (findInputs() == srcAttempts.size()) {
           // double checked after lock

http://git-wip-us.apache.org/repos/asf/tez/blob/86309f93/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
index 6638633..b3382ea 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
@@ -38,6 +38,7 @@ public class InputHost {
   private final int port;
   private final int srcPhysicalIndex;
   private final String identifier;
+  private String additionalInfo;
 
   private final BlockingQueue<InputAttemptIdentifier> inputs = new LinkedBlockingQueue<InputAttemptIdentifier>();
 
@@ -64,6 +65,14 @@ public class InputHost {
     return this.identifier;
   }
 
+  public void setAdditionalInfo(String additionalInfo) {
+    this.additionalInfo = additionalInfo;
+  }
+
+  public String getAdditionalInfo() {
+    return (additionalInfo == null) ? "" : additionalInfo;
+  }
+
   public int getSrcPhysicalIndex() {
     return this.srcPhysicalIndex;
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/86309f93/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
index 5d1da7c..5cc1ef0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
@@ -317,7 +317,6 @@ public class ShuffleManager implements FetcherCallback {
                 LOG.debug("Processing pending host: " + inputHost.toDetailedString());
               }
               if (inputHost.getNumPendingInputs() > 0 && !isShutdown.get()) {
-                LOG.info("Scheduling fetch for inputHost: " + inputHost.getIdentifier());
                 Fetcher fetcher = constructFetcherForHost(inputHost, conf);
                 runningFetchers.add(fetcher);
                 if (isShutdown.get()) {
@@ -404,6 +403,7 @@ public class ShuffleManager implements FetcherCallback {
     fetcherBuilder.assignWork(inputHost.getHost(), inputHost.getPort(),
         inputHost.getSrcPhysicalIndex(), pendingInputsForHost);
     LOG.info("Created Fetcher for host: " + inputHost.getHost()
+        + ", info: " + inputHost.getAdditionalInfo()
         + ", with inputs: " + pendingInputsForHost);
     return fetcherBuilder.build();
   }
@@ -817,6 +817,7 @@ public class ShuffleManager implements FetcherCallback {
           for (InputAttemptIdentifier input : pendingInputs) {
             inputHost.addKnownInput(input);
           }
+          inputHost.setAdditionalInfo(result.getAdditionalInfo());
           pendingHosts.add(inputHost);
         }
         doBookKeepingForFetcherComplete();


Mime
View raw message