hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject git commit: YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong.
Date Sun, 26 Oct 2014 20:00:46 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6 259db7a72 -> ab7a21569


YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by
Xuan Gong.

(cherry picked from commit caecd9fffe7c6216be31f3ab65349182045451fa)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab7a2156
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab7a2156
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab7a2156

Branch: refs/heads/branch-2.6
Commit: ab7a21569cbceee482aa8b009e4a2d8a2f766434
Parents: 259db7a
Author: Zhijie Shen <zjshen@apache.org>
Authored: Sun Oct 26 12:57:31 2014 -0700
Committer: Zhijie Shen <zjshen@apache.org>
Committed: Sun Oct 26 13:00:26 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                             | 3 +++
 .../hadoop/yarn/logaggregation/AggregatedLogFormat.java     | 6 ++++++
 .../hadoop/yarn/logaggregation/TestAggregatedLogFormat.java | 9 +++++++++
 3 files changed, 18 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab7a2156/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 30f87e2..a613c80 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,6 +694,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port.
     (Naganarasimha G R via xgong)
 
+    YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
+    (Xuan Gong via zjshen)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab7a2156/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 035d61d..22219be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -213,6 +213,12 @@ public class AggregatedLogFormat {
       Collections.sort(fileList);
 
       for (File logFile : fileList) {
+        // We only aggregate top level files.
+        // Ignore anything inside sub-folders.
+        if (logFile.isDirectory()) {
+          LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it.");
+          continue;
+        }
 
         FileInputStream in = null;
         try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab7a2156/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index fa9de74..1d607b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -194,6 +194,14 @@ public class TestAggregatedLogFormat {
 
     int numChars = 80000;
 
+    // create a sub-folder under srcFilePath
+    // and create file logs in this sub-folder.
+    // We only aggregate top level files.
+    // So, this log file should be ignored.
+    Path subDir = new Path(srcFilePath, "subDir");
+    fs.mkdirs(subDir);
+    writeSrcFile(subDir, "logs", numChars);
+
     // create file stderr and stdout in containerLogDir
     writeSrcFile(srcFilePath, "stderr", numChars);
     writeSrcFile(srcFilePath, "stdout", numChars);
@@ -238,6 +246,7 @@ public class TestAggregatedLogFormat {
             + "\nLog Contents:\n".length() + numChars;
     Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
     Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
+    Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
     Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
     Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
     


Mime
View raw message