hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From junping...@apache.org
Subject hadoop git commit: YARN-7083. Log aggregation deletes/renames while file is open. Contributed by Jason Lowe.
Date Tue, 29 Aug 2017 20:22:45 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8.2 a766b72ba -> 12e35f4ef


YARN-7083. Log aggregation deletes/renames while file is open. Contributed by Jason Lowe.

(cherry picked from commit 79294b5f329317c30775f8b4dc61440f4b6c24cd)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12e35f4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12e35f4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12e35f4e

Branch: refs/heads/branch-2.8.2
Commit: 12e35f4ef14071b8a0626ba7025dbcd8c23ad2dc
Parents: a766b72
Author: Junping Du <junping_du@apache.org>
Authored: Tue Aug 29 13:18:49 2017 -0700
Committer: Junping Du <junping_du@apache.org>
Committed: Tue Aug 29 13:20:05 2017 -0700

----------------------------------------------------------------------
 .../logaggregation/AppLogAggregatorImpl.java    | 83 ++++++++++----------
 1 file changed, 42 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/12e35f4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index df972f0..e2975b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -320,53 +320,54 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
     logAggregationTimes++;
     String diagnosticMessage = "";
     boolean logAggregationSucceedInThisCycle = true;
-    try (LogWriter writer = new LogWriter()){
-      try {
-        writer.initialize(this.conf, this.remoteNodeTmpLogFileForApp,
-            this.userUgi);
-        // Write ACLs once when the writer is created.
-        writer.writeApplicationACLs(appAcls);
-        writer.writeApplicationOwner(this.userUgi.getShortUserName());
-      } catch (IOException e1) {
-        logAggregationSucceedInThisCycle = false;
-        LOG.error("Cannot create writer for app " + this.applicationId
-            + ". Skip log upload this time. ", e1);
-        return;
-      }
-
+    try {
       boolean uploadedLogsInThisCycle = false;
-      for (ContainerId container : pendingContainerInThisCycle) {
-        ContainerLogAggregator aggregator = null;
-        if (containerLogAggregators.containsKey(container)) {
-          aggregator = containerLogAggregators.get(container);
-        } else {
-          aggregator = new ContainerLogAggregator(container);
-          containerLogAggregators.put(container, aggregator);
-        }
-        Set<Path> uploadedFilePathsInThisCycle =
-            aggregator.doContainerLogAggregation(writer, appFinished,
-            finishedContainers.contains(container));
-        if (uploadedFilePathsInThisCycle.size() > 0) {
-          uploadedLogsInThisCycle = true;
-          this.delService.delete(this.userUgi.getShortUserName(), null,
-              uploadedFilePathsInThisCycle
-                  .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
+      try (LogWriter writer = new LogWriter()){
+        try {
+          writer.initialize(this.conf, this.remoteNodeTmpLogFileForApp,
+              this.userUgi);
+          // Write ACLs once when the writer is created.
+          writer.writeApplicationACLs(appAcls);
+          writer.writeApplicationOwner(this.userUgi.getShortUserName());
+        } catch (IOException e1) {
+          logAggregationSucceedInThisCycle = false;
+          LOG.error("Cannot create writer for app " + this.applicationId
+              + ". Skip log upload this time. ", e1);
+          return;
         }
 
-        // This container is finished, and all its logs have been uploaded,
-        // remove it from containerLogAggregators.
-        if (finishedContainers.contains(container)) {
-          containerLogAggregators.remove(container);
+        for (ContainerId container : pendingContainerInThisCycle) {
+          ContainerLogAggregator aggregator = null;
+          if (containerLogAggregators.containsKey(container)) {
+            aggregator = containerLogAggregators.get(container);
+          } else {
+            aggregator = new ContainerLogAggregator(container);
+            containerLogAggregators.put(container, aggregator);
+          }
+          Set<Path> uploadedFilePathsInThisCycle =
+              aggregator.doContainerLogAggregation(writer, appFinished,
+              finishedContainers.contains(container));
+          if (uploadedFilePathsInThisCycle.size() > 0) {
+            uploadedLogsInThisCycle = true;
+            this.delService.delete(this.userUgi.getShortUserName(), null,
+                uploadedFilePathsInThisCycle
+                    .toArray(new Path[uploadedFilePathsInThisCycle.size()]));
+          }
+
+          // This container is finished, and all its logs have been uploaded,
+          // remove it from containerLogAggregators.
+          if (finishedContainers.contains(container)) {
+            containerLogAggregators.remove(container);
+          }
         }
-      }
 
-      // Before upload logs, make sure the number of existing logs
-      // is smaller than the configured NM log aggregation retention size.
-      if (uploadedLogsInThisCycle && logAggregationInRolling) {
-        cleanOldLogs();
-        cleanupOldLogTimes++;
+        // Before upload logs, make sure the number of existing logs
+        // is smaller than the configured NM log aggregation retention size.
+        if (uploadedLogsInThisCycle && logAggregationInRolling) {
+          cleanOldLogs();
+          cleanupOldLogTimes++;
+        }
       }
-
       long currentTime = System.currentTimeMillis();
       final Path renamedPath = this.rollingMonitorInterval <= 0
               ? remoteNodeLogFileForApp : new Path(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message