hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From viraj...@apache.org
Subject [15/50] [abbrv] hadoop git commit: YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)
Date Tue, 24 Oct 2017 18:31:49 GMT
YARN-7261. Add debug message for better download latency monitoring. (Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0799fde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0799fde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0799fde3

Branch: refs/heads/HDFS-9806
Commit: 0799fde35e7f3b9e8a85284ac0b30f6bdcbffad1
Parents: b61144a
Author: Yufei Gu <yufei@apache.org>
Authored: Fri Oct 20 09:59:07 2017 -0700
Committer: Yufei Gu <yufei@apache.org>
Committed: Fri Oct 20 10:00:13 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/yarn/util/FSDownload.java    | 17 +++++++++++++++--
 .../localizer/ResourceLocalizationService.java     | 11 ++++++++++-
 2 files changed, 25 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index f34c16c..6e59574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -350,6 +350,11 @@ public class FSDownload implements Callable<Path> {
     } catch (URISyntaxException e) {
       throw new IOException("Invalid resource", e);
     }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Starting to download " + sCopy);
+    }
+
     createDir(destDirPath, cachePerms);
     final Path dst_work = new Path(destDirPath + "_tmp");
     createDir(dst_work, cachePerms);
@@ -364,6 +369,11 @@ public class FSDownload implements Callable<Path> {
       unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
       changePermissions(dFinal.getFileSystem(conf), dFinal);
       files.rename(dst_work, destDirPath, Rename.OVERWRITE);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("File has been downloaded to " +
+            new Path(destDirPath, sCopy.getName()));
+      }
     } catch (Exception e) {
       try {
         files.delete(destDirPath, true);
@@ -409,8 +419,11 @@ public class FSDownload implements Callable<Path> {
       // APPLICATION:
       perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
     }
-    LOG.debug("Changing permissions for path " + path
-        + " to perm " + perm);
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Changing permissions for path " + path + " to perm " + perm);
+    }
+
     final FsPermission fPerm = perm;
     if (null == userUgi) {
       files.setPermission(path, perm);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0799fde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index c37f2e3..0e7861f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -859,7 +859,7 @@ public class ResourceLocalizationService extends CompositeService
       // TODO handle failures, cancellation, requests by other containers
       LocalizedResource rsrc = request.getResource();
       LocalResourceRequest key = rsrc.getRequest();
-      LOG.info("Downloading public rsrc:" + key);
+      LOG.info("Downloading public resource: " + key);
       /*
        * Here multiple containers may request the same resource. So we need
        * to start downloading only when
@@ -918,8 +918,17 @@ public class ResourceLocalizationService extends CompositeService
                 + " Either queue is full or threadpool is shutdown.", re);
           }
         } else {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Skip downloading resource: " + key + " since it's in"
+                + " state: " + rsrc.getState());
+          }
           rsrc.unlock();
         }
+      } else {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Skip downloading resource: " + key + " since it is locked"
+              + " by other threads");
+        }
       }
     }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message