hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject svn commit: r1487498 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/ hadoop-yarn/hadoop-yarn-server...
Date Wed, 29 May 2013 14:25:04 GMT
Author: jlowe
Date: Wed May 29 14:25:04 2013
New Revision: 1487498

URL: http://svn.apache.org/r1487498
Log:
YARN-512. Log aggregation root directory check is more expensive than it needs to be. Contributed
by Maysam Yabandeh

Modified:
    hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
    hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java

Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1487498&r1=1487497&r2=1487498&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed May 29 14:25:04 2013
@@ -223,6 +223,9 @@ Release 2.0.5-beta - UNRELEASED
 
   OPTIMIZATIONS
 
+    YARN-512. Log aggregation root directory check is more expensive than it
+    needs to be. (Maysam Yabandeh via jlowe)
+
   BUG FIXES
 
     YARN-383. AMRMClientImpl should handle null rmClient in stop()

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1487498&r1=1487497&r2=1487498&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
Wed May 29 14:25:04 2013
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Map;
@@ -163,36 +164,31 @@ public class LogAggregationService exten
   }
   
   void verifyAndCreateRemoteLogDir(Configuration conf) {
-    // Checking the existance of the TLD
+    // Checking the existence of the TLD
     FileSystem remoteFS = null;
     try {
       remoteFS = FileSystem.get(conf);
     } catch (IOException e) {
       throw new YarnException("Unable to get Remote FileSystem instance", e);
     }
-    boolean remoteExists = false;
+    boolean remoteExists = true;
     try {
-      remoteExists = remoteFS.exists(this.remoteRootLogDir);
+      FsPermission perms =
+          remoteFS.getFileStatus(this.remoteRootLogDir).getPermission();
+      if (!perms.equals(TLDIR_PERMISSIONS)) {
+        LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
+            + "] already exist, but with incorrect permissions. "
+            + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
+            + "]." + " The cluster may have problems with multiple users.");
+      }
+    } catch (FileNotFoundException e) {
+      remoteExists = false;
     } catch (IOException e) {
-      throw new YarnException("Failed to check for existence of remoteLogDir ["
-          + this.remoteRootLogDir + "]", e);
+      throw new YarnException(
+          "Failed to check permissions for dir ["
+              + this.remoteRootLogDir + "]", e);
     }
-    if (remoteExists) {
-      try {
-        FsPermission perms =
-            remoteFS.getFileStatus(this.remoteRootLogDir).getPermission();
-        if (!perms.equals(TLDIR_PERMISSIONS)) {
-          LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
-              + "] already exist, but with incorrect permissions. "
-              + "Expected: [" + TLDIR_PERMISSIONS + "], Found: [" + perms
-              + "]." + " The cluster may have problems with multiple users.");
-        }
-      } catch (IOException e) {
-        throw new YarnException(
-            "Failed to check permissions for dir ["
-                + this.remoteRootLogDir + "]", e);
-      }
-    } else {
+    if (!remoteExists) {
       LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir
           + "] does not exist. Attempting to create it.");
       try {

Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1487498&r1=1487497&r2=1487498&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
(original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
Wed May 29 14:25:04 2013
@@ -465,6 +465,30 @@ public class TestLogAggregationService e
     logAggregationService.stop();
   }
   
+  
+  @Test
+  public void testVerifyAndCreateRemoteDirNonExistence()
+      throws Exception {
+    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
+    File aNewFile = new File(String.valueOf("tmp"+System.currentTimeMillis()));
+    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, 
+        aNewFile.getAbsolutePath());
+    
+    DrainDispatcher dispatcher = createDispatcher();
+    LogAggregationService logAggregationService = spy(
+        new LogAggregationService(dispatcher, this.context, this.delSrvc,
+                                  super.dirsHandler));
+    logAggregationService.init(this.conf);
+    boolean existsBefore = aNewFile.exists();
+    assertTrue("The new file already exists!", !existsBefore);
+
+    logAggregationService.verifyAndCreateRemoteLogDir(this.conf);
+    
+    boolean existsAfter = aNewFile.exists();
+    assertTrue("The new aggregate file is not successfully created", existsAfter);
+    aNewFile.delete(); //housekeeping
+  }
+  
   @Test
   @SuppressWarnings("unchecked")
   public void testLogAggregationInitAppFailsWithoutKillingNM() throws Exception {



Mime
View raw message