hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjs...@apache.org
Subject git commit: YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong.
Date Sun, 26 Oct 2014 19:59:48 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b1e17d2da -> cb3974524


YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by
Xuan Gong.

(cherry picked from commit caecd9fffe7c6216be31f3ab65349182045451fa)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb397452
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb397452
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb397452

Branch: refs/heads/branch-2
Commit: cb3974524c5b265e70d94d0d40b24e2e9224f050
Parents: b1e17d2
Author: Zhijie Shen <zjshen@apache.org>
Authored: Sun Oct 26 12:57:31 2014 -0700
Committer: Zhijie Shen <zjshen@apache.org>
Committed: Sun Oct 26 12:59:29 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                             | 3 +++
 .../hadoop/yarn/logaggregation/AggregatedLogFormat.java     | 6 ++++++
 .../hadoop/yarn/logaggregation/TestAggregatedLogFormat.java | 9 +++++++++
 3 files changed, 18 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb397452/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0c9e525..74d1dff 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -730,6 +730,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port.
     (Naganarasimha G R via xgong)
 
+    YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
+    (Xuan Gong via zjshen)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb397452/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 035d61d..22219be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -213,6 +213,12 @@ public class AggregatedLogFormat {
       Collections.sort(fileList);
 
       for (File logFile : fileList) {
+        // We only aggregate top level files.
+        // Ignore anything inside sub-folders.
+        if (logFile.isDirectory()) {
+          LOG.warn(logFile.getAbsolutePath() + " is a directory. Ignore it.");
+          continue;
+        }
 
         FileInputStream in = null;
         try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb397452/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index fa9de74..1d607b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -194,6 +194,14 @@ public class TestAggregatedLogFormat {
 
     int numChars = 80000;
 
+    // create a sub-folder under srcFilePath
+    // and create file logs in this sub-folder.
+    // We only aggregate top level files.
+    // So, this log file should be ignored.
+    Path subDir = new Path(srcFilePath, "subDir");
+    fs.mkdirs(subDir);
+    writeSrcFile(subDir, "logs", numChars);
+
     // create file stderr and stdout in containerLogDir
     writeSrcFile(srcFilePath, "stderr", numChars);
     writeSrcFile(srcFilePath, "stdout", numChars);
@@ -238,6 +246,7 @@ public class TestAggregatedLogFormat {
             + "\nLog Contents:\n".length() + numChars;
     Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
     Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
+    Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
     Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
     Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
     


Mime
View raw message