tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject tez git commit: TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option (rbalamohan)
Date Sun, 16 Aug 2015 23:00:53 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 4bf4c4409 -> 8e94986f0


TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option (rbalamohan)

(cherry picked from commit a405ce95762ffc314269a0428e260b966a1627da)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/8e94986f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/8e94986f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/8e94986f

Branch: refs/heads/branch-0.5
Commit: 8e94986f08c0c46b81728ba9a9ac2309db3f2c04
Parents: 4bf4c44
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Mon Aug 17 04:23:21 2015 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Mon Aug 17 04:32:48 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                              |  1 +
 .../tez/runtime/library/common/shuffle/FetchResult.java  | 11 +++++++++++
 .../tez/runtime/library/common/shuffle/Fetcher.java      |  6 +-----
 .../tez/runtime/library/common/shuffle/InputHost.java    |  9 +++++++++
 .../library/common/shuffle/impl/ShuffleManager.java      |  3 ++-
 5 files changed, 24 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7831fb9..f519ee6 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
   TEZ-2552. CRC errors can cause job to run for very long time in large jobs.
 
 ALL CHANGES:
+  TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option
   TEZ-2630. TezChild receives IP address instead of FQDN. (hitesh)
   TEZ-2635. Limit number of attempts being downloaded in unordered fetch.
   TEZ-2636. MRInput and MultiMRInput should work for cases when there are 0 physical inputs.

http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
index 1c39a24..d9595f0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java
@@ -43,13 +43,20 @@ public class FetchResult {
   private final int port;
   private final int partition;
   private final Iterable<InputAttemptIdentifier> pendingInputs;
+  private final String additionalInfo;
 
   public FetchResult(String host, int port, int partition,
       Iterable<InputAttemptIdentifier> pendingInputs) {
+    this(host, port, partition, pendingInputs, null);
+  }
+
+  public FetchResult(String host, int port, int partition,
+      Iterable<InputAttemptIdentifier> pendingInputs, String additionalInfo) {
     this.host = host;
     this.port = port;
     this.partition = partition;
     this.pendingInputs = pendingInputs;
+    this.additionalInfo = additionalInfo;
   }
 
   public String getHost() {
@@ -67,4 +74,8 @@ public class FetchResult {
   public Iterable<InputAttemptIdentifier> getPendingInputs() {
     return pendingInputs;
   }
+
+  public String getAdditionalInfo() {
+    return additionalInfo;
+  }
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
index 6875966..e4cf071 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java
@@ -208,8 +208,6 @@ public class Fetcher implements Callable<FetchResult> {
       if (!multiplex) {
         throw new IOException("server didn't return all expected map outputs: "
             + remaining.size() + " left.");
-      } else {
-        LOG.info("Shared fetch failed to return " + remaining.size() + " inputs on this try");
       }
     }
 
@@ -355,10 +353,8 @@ public class Fetcher implements Callable<FetchResult> {
       lock = getLock();
       if (lock == null) {
         // re-queue until we get a lock
-        LOG.info("Requeuing " + host + ":" + port
-            + " downloads because we didn't get a lock");
         return new HostFetchResult(new FetchResult(host, port, partition,
-            remaining), null, false);
+            remaining, "Requeuing as we didn't get a lock"), null, false);
       } else {
         if (findInputs() == srcAttempts.size()) {
           // double checked after lock

http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
index 6638633..b3382ea 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java
@@ -38,6 +38,7 @@ public class InputHost {
   private final int port;
   private final int srcPhysicalIndex;
   private final String identifier;
+  private String additionalInfo;
 
   private final BlockingQueue<InputAttemptIdentifier> inputs = new LinkedBlockingQueue<InputAttemptIdentifier>();
 
@@ -64,6 +65,14 @@ public class InputHost {
     return this.identifier;
   }
 
+  public void setAdditionalInfo(String additionalInfo) {
+    this.additionalInfo = additionalInfo;
+  }
+
+  public String getAdditionalInfo() {
+    return (additionalInfo == null) ? "" : additionalInfo;
+  }
+
   public int getSrcPhysicalIndex() {
     return this.srcPhysicalIndex;
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/8e94986f/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
index b032bdc..d93de20 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java
@@ -310,7 +310,6 @@ public class ShuffleManager implements FetcherCallback {
                 LOG.debug("Processing pending host: " + inputHost.toDetailedString());
               }
               if (inputHost.getNumPendingInputs() > 0 && !isShutdown.get()) {
-                LOG.info("Scheduling fetch for inputHost: " + inputHost.getIdentifier());
                 Fetcher fetcher = constructFetcherForHost(inputHost, conf);
                 runningFetchers.add(fetcher);
                 if (isShutdown.get()) {
@@ -396,6 +395,7 @@ public class ShuffleManager implements FetcherCallback {
     fetcherBuilder.assignWork(inputHost.getHost(), inputHost.getPort(),
         inputHost.getSrcPhysicalIndex(), pendingInputsForHost);
     LOG.info("Created Fetcher for host: " + inputHost.getHost()
+        + ", info: " + inputHost.getAdditionalInfo()
         + ", with inputs: " + pendingInputsForHost);
     return fetcherBuilder.build();
   }
@@ -799,6 +799,7 @@ public class ShuffleManager implements FetcherCallback {
           for (InputAttemptIdentifier input : pendingInputs) {
             inputHost.addKnownInput(input);
           }
+          inputHost.setAdditionalInfo(result.getAdditionalInfo());
           pendingHosts.add(inputHost);
         }
         doBookKeepingForFetcherComplete();


Mime
View raw message