hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject hadoop git commit: YARN-4842. Fixed "yarn logs" command to guess (and thus not require) the appOwner argument when viewing another user's logs. Contributed by Ram Venkatesh and Xuan Gong.
Date Tue, 10 May 2016 05:51:19 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 007d6d1d5 -> e0d6a9632


YARN-4842. Fixed "yarn logs" command to guess (and thus not require) the appOwner argument
when viewing another user's logs. Contributed by Ram Venkatesh and Xuan Gong.

(cherry picked from commit 87f5e351337a905af5215af76c72b9312616cd4f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0d6a963
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0d6a963
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0d6a963

Branch: refs/heads/branch-2
Commit: e0d6a9632daaa437487e52a6e0d1abeb84f7994a
Parents: 007d6d1
Author: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Authored: Mon May 9 22:33:14 2016 -0700
Committer: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Committed: Mon May 9 22:47:58 2016 -0700

----------------------------------------------------------------------
 .../apache/hadoop/yarn/client/cli/LogsCLI.java  |  55 +++--
 .../hadoop/yarn/client/cli/TestLogsCLI.java     | 203 ++++++++++++++++---
 .../yarn/logaggregation/LogCLIHelpers.java      |  82 +++++++-
 3 files changed, 290 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d6a963/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 487b694..926ba79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -151,14 +151,12 @@ public class LogsCLI extends Configured implements Tool {
     LogCLIHelpers logCliHelper = new LogCLIHelpers();
     logCliHelper.setConf(getConf());
 
-    if (appOwner == null || appOwner.isEmpty()) {
-      appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
-    }
-
     boolean appStateObtainedSuccessfully = true;
     YarnApplicationState appState = YarnApplicationState.NEW;
+    ApplicationReport appReport = null;
     try {
-      appState = getApplicationState(appId);
+      appReport = getApplicationReport(appId);
+      appState = appReport.getYarnApplicationState();
       if (appState == YarnApplicationState.NEW
           || appState == YarnApplicationState.NEW_SAVING
           || appState == YarnApplicationState.SUBMITTED) {
@@ -171,6 +169,16 @@ public class LogsCLI extends Configured implements Tool {
           + " Attempting to fetch logs directly from the filesystem.");
     }
 
+    if (appOwner == null || appOwner.isEmpty()) {
+      appOwner = guessAppOwner(appReport, appId);
+      if (appOwner == null) {
+        System.err.println("Can not find the appOwner. "
+            + "Please specify the correct appOwner");
+        System.err.println("Could not locate application logs for " + appId);
+        return -1;
+      }
+    }
+
     if (showMetaInfo) {
       return showMetaInfo(appState, appStateObtainedSuccessfully,
           logCliHelper, appId, containerIdStr, nodeAddress, appOwner);
@@ -201,6 +209,10 @@ public class LogsCLI extends Configured implements Tool {
       if (nodeAddress == null) {
         resultCode =
             logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out);
+        if (resultCode == -1) {
+          System.err.println("Can not find the logs for the application: "
+              + appId + " with the appOwner: " + appOwner);
+        }
       } else {
         System.err.println("Should at least provide ContainerId!");
         printHelpMessage(printOpts);
@@ -210,13 +222,12 @@ public class LogsCLI extends Configured implements Tool {
     return resultCode;
   }
 
-  private YarnApplicationState getApplicationState(ApplicationId appId)
+  private ApplicationReport getApplicationReport(ApplicationId appId)
       throws IOException, YarnException {
     YarnClient yarnClient = createYarnClient();
 
     try {
-      ApplicationReport appReport = yarnClient.getApplicationReport(appId);
-      return appReport.getYarnApplicationState();
+      return yarnClient.getApplicationReport(appId);
     } finally {
       yarnClient.close();
     }
@@ -693,11 +704,12 @@ public class LogsCLI extends Configured implements Tool {
             amContainersList, logFiles, logCliHelper, appOwner, true);
       } else {
         System.err.println("Can not get AMContainers logs for "
-            + "the application:" + appId);
-        System.err.println("This application:" + appId + " is finished."
-            + " Please enable the application history service. Or Using "
-            + "yarn logs -applicationId <appId> -containerId <containerId> "
-            + "--nodeAddress <nodeHttpAddress> to get the container logs");
+            + "the application:" + appId + " with the appOwner:" + appOwner);
+        System.err.println("This application:" + appId + " has finished."
+            + " Please enable the application-history service or explicitly"
+            + " use 'yarn logs -applicationId <appId> "
+            + "-containerId <containerId> --nodeAddress <nodeHttpAddress>' "
+            + "to get the container logs.");
         return -1;
       }
     }
@@ -750,7 +762,8 @@ public class LogsCLI extends Configured implements Tool {
             appOwner);
       } else if (!isApplicationFinished(appState)) {
         System.err.println("Unable to get logs for this container:"
-            + containerIdStr + "for the application:" + appIdStr);
+            + containerIdStr + "for the application:" + appIdStr
+            + " with the appOwner: " + appOwner);
         System.err.println("The application: " + appIdStr
             + " is still running, and we can not get Container report "
             + "for the container: " + containerIdStr +". Please try later "
@@ -821,4 +834,18 @@ public class LogsCLI extends Configured implements Tool {
       return isAppFinished;
     }
   }
+
+  private String guessAppOwner(ApplicationReport appReport,
+      ApplicationId appId) throws IOException {
+    String appOwner = null;
+    if (appReport != null) {
+      //always use the app owner from the app report if possible
+      appOwner = appReport.getUser();
+    } else {
+      appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
+      appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(
+          appId, appOwner, getConf());
+    }
+    return appOwner;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d6a963/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index b950764..4f1c629 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -42,11 +42,11 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
-import org.junit.Assert;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -55,22 +55,21 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
 import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
 import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 public class TestLogsCLI {
+
   ByteArrayOutputStream sysOutStream;
   private PrintStream sysOut;
-  
+
   ByteArrayOutputStream sysErrStream;
   private PrintStream sysErr;
 
@@ -79,7 +78,7 @@ public class TestLogsCLI {
     sysOutStream = new ByteArrayOutputStream();
     sysOut =  new PrintStream(sysOutStream);
     System.setOut(sysOut);
-    
+
     sysErrStream = new ByteArrayOutputStream();
     sysErr = new PrintStream(sysErrStream);
     System.setErr(sysErr);
@@ -91,16 +90,18 @@ public class TestLogsCLI {
     conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
     LogCLIHelpers cliHelper = new LogCLIHelpers();
     cliHelper.setConf(conf);
-    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
+    YarnClient mockYarnClient = createMockYarnClient(
+        YarnApplicationState.FINISHED,
+        UserGroupInformation.getCurrentUser().getShortUserName());
     LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
     dumper.setConf(conf);
-    
+
     // verify dumping a non-existent application's logs returns a failure code
     int exitCode = dumper.run( new String[] {
         "-applicationId", "application_0_0" } );
     assertTrue("Should return an error code", exitCode != 0);
-    
-    // verify dumping a non-existent container log is a failure code 
+
+    // verify dumping a non-existent container log is a failure code
     exitCode = cliHelper.dumpAContainersLogs("application_0_0", "container_0_0",
         "nonexistentnode:1234", "nobody");
     assertTrue("Should return an error code", exitCode != 0);
@@ -109,10 +110,12 @@ public class TestLogsCLI {
   @Test(timeout = 5000l)
   public void testInvalidApplicationId() throws Exception {
     Configuration conf = new YarnConfiguration();
-    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
+    YarnClient mockYarnClient = createMockYarnClient(
+        YarnApplicationState.FINISHED,
+        UserGroupInformation.getCurrentUser().getShortUserName());
     LogsCLI cli = new LogsCLIForTest(mockYarnClient);
     cli.setConf(conf);
-    
+
     int exitCode = cli.run( new String[] { "-applicationId", "not_an_app_id"});
     assertTrue(exitCode == -1);
     assertTrue(sysErrStream.toString().startsWith("Invalid ApplicationId specified"));
@@ -137,7 +140,9 @@ public class TestLogsCLI {
   @Test(timeout = 5000l)
   public void testHelpMessage() throws Exception {
     Configuration conf = new YarnConfiguration();
-    YarnClient mockYarnClient = createMockYarnClient(YarnApplicationState.FINISHED);
+    YarnClient mockYarnClient = createMockYarnClient(
+        YarnApplicationState.FINISHED,
+        UserGroupInformation.getCurrentUser().getShortUserName());
     LogsCLI dumper = new LogsCLIForTest(mockYarnClient);
     dumper.setConf(conf);
 
@@ -187,7 +192,7 @@ public class TestLogsCLI {
     String appReportStr = baos.toString("UTF-8");
     Assert.assertEquals(appReportStr, sysOutStream.toString());
   }
-  
+
   @Test (timeout = 15000)
   public void testFetchApplictionLogs() throws Exception {
     String remoteLogRootDir = "target/logs/";
@@ -200,13 +205,13 @@ public class TestLogsCLI {
     FileSystem fs = FileSystem.get(configuration);
 
     UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-    ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
     ApplicationAttemptId appAttemptId =
-        ApplicationAttemptIdPBImpl.newInstance(appId, 1);
-    ContainerId containerId0 = ContainerIdPBImpl.newContainerId(appAttemptId, 0);
-    ContainerId containerId1 = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
-    ContainerId containerId2 = ContainerIdPBImpl.newContainerId(appAttemptId, 2);
-    ContainerId containerId3 = ContainerIdPBImpl.newContainerId(appAttemptId, 3);
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId0 = ContainerId.newContainerId(appAttemptId, 0);
+    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
+    ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2);
+    ContainerId containerId3 = ContainerId.newContainerId(appAttemptId, 3);
     NodeId nodeId = NodeId.newInstance("localhost", 1234);
 
     // create local logs
@@ -222,6 +227,7 @@ public class TestLogsCLI {
       fs.delete(appLogsDir, true);
     }
     assertTrue(fs.mkdirs(appLogsDir));
+
     List<String> rootLogDirs = Arrays.asList(rootLogDir);
 
     List<String> logTypes = new ArrayList<String>();
@@ -258,7 +264,8 @@ public class TestLogsCLI {
       containerId3, path, fs);
 
     YarnClient mockYarnClient =
-        createMockYarnClient(YarnApplicationState.FINISHED);
+        createMockYarnClient(
+            YarnApplicationState.FINISHED, ugi.getShortUserName());
     LogsCLI cli = new LogsCLIForTest(mockYarnClient);
     cli.setConf(configuration);
 
@@ -349,6 +356,143 @@ public class TestLogsCLI {
   }
 
   @Test (timeout = 15000)
+  public void testFetchApplictionLogsAsAnotherUser() throws Exception {
+    String remoteLogRootDir = "target/logs/";
+    String rootLogDir = "target/LocalLogs";
+
+    String testUser = "test";
+    UserGroupInformation testUgi = UserGroupInformation
+        .createRemoteUser(testUser);
+
+    Configuration configuration = new Configuration();
+    configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
+    configuration
+        .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
+    configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+    configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
+    FileSystem fs = FileSystem.get(configuration);
+
+    ApplicationId appId = ApplicationId.newInstance(0, 1);
+    ApplicationAttemptId appAttemptId =
+        ApplicationAttemptId.newInstance(appId, 1);
+    ContainerId containerId = ContainerId
+        .newContainerId(appAttemptId, 1);
+    NodeId nodeId = NodeId.newInstance("localhost", 1234);
+
+    try {
+      Path rootLogDirPath = new Path(rootLogDir);
+      if (fs.exists(rootLogDirPath)) {
+        fs.delete(rootLogDirPath, true);
+      }
+      assertTrue(fs.mkdirs(rootLogDirPath));
+
+      // create local app dir for app
+      final Path appLogsDir = new Path(rootLogDirPath, appId.toString());
+      if (fs.exists(appLogsDir)) {
+        fs.delete(appLogsDir, true);
+      }
+      assertTrue(fs.mkdirs(appLogsDir));
+
+      List<String> rootLogDirs = Arrays.asList(rootLogDir);
+      List<String> logTypes = new ArrayList<String>();
+      logTypes.add("syslog");
+
+      // create container logs in localLogDir for app
+      createContainerLogInLocalDir(appLogsDir, containerId, fs, logTypes);
+
+      // create the remote app dir for app
+      // but for a different user testUser"
+      Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId);
+      if (fs.exists(path)) {
+        fs.delete(path, true);
+      }
+      assertTrue(fs.mkdirs(path));
+
+      // upload container logs for app into remote dir
+      uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
+          nodeId, containerId, path, fs);
+
+      YarnClient mockYarnClient = createMockYarnClient(
+          YarnApplicationState.FINISHED, testUgi.getShortUserName());
+      LogsCLI cli = new LogsCLIForTest(mockYarnClient);
+      cli.setConf(configuration);
+
+      // Verify that we can get the application logs by specifying
+      // a correct appOwner
+      int exitCode = cli.run(new String[] {
+          "-applicationId", appId.toString(),
+          "-appOwner", testUser});
+      assertTrue(exitCode == 0);
+      assertTrue(sysOutStream.toString().contains(
+          "Hello " + containerId + " in syslog!"));
+      sysOutStream.reset();
+
+      // Verify that we can not get the application logs
+      // if an invalid user is specified
+      exitCode = cli.run(new String[] {
+          "-applicationId", appId.toString(),
+          "-appOwner", "invalid"});
+      assertTrue(exitCode == -1);
+      assertTrue(sysErrStream.toString().contains("Can not find the logs "
+          + "for the application: " + appId.toString()));
+      sysErrStream.reset();
+
+      // Verify that we do not specify appOwner, and can not
+      // get appReport from RM, we still can figure out the appOwner
+      // and can get app logs successfully.
+      YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
+      cli = new LogsCLIForTest(mockYarnClient2);
+      cli.setConf(configuration);
+      exitCode = cli.run(new String[] {
+          "-applicationId", appId.toString()});
+      assertTrue(exitCode == 0);
+      assertTrue(sysOutStream.toString().contains("Hello "
+          + containerId + " in syslog!"));
+      sysOutStream.reset();
+
+      // Verify that we could get the err message "Can not find the appOwner"
+      // if we do not specify the appOwner, can not get appReport, and
+      // the app does not exist in remote dir.
+      ApplicationId appId2 = ApplicationId.newInstance(
+          System.currentTimeMillis(), 2);
+      exitCode = cli.run(new String[] {
+          "-applicationId", appId2.toString()});
+      assertTrue(exitCode == -1);
+      assertTrue(sysErrStream.toString().contains(
+          "Can not find the appOwner"));
+      sysErrStream.reset();
+
+      // Verify that we could not get appOwner
+      // because we don't have file-system permissions
+      ApplicationId appTest = ApplicationId.newInstance(
+          System.currentTimeMillis(), 1000);
+      String priorityUser = "priority";
+      Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser
+          + "/logs/" + appTest);
+      if (fs.exists(pathWithoutPerm)) {
+        fs.delete(pathWithoutPerm, true);
+      }
+      // The user will not have read permission for this directory.
+      // To mimic the scenario that the user can not get file status
+      FsPermission permission = FsPermission
+          .createImmutable((short) 01300);
+      assertTrue(fs.mkdirs(pathWithoutPerm, permission));
+
+      exitCode = cli.run(new String[] {
+          "-applicationId", appTest.toString()});
+      assertTrue(exitCode == -1);
+      assertTrue(sysErrStream.toString().contains(
+        "Guessed logs' owner is " + priorityUser + " and current user "
+            + UserGroupInformation.getCurrentUser().getUserName()
+            + " does not have permission to access"));
+      sysErrStream.reset();
+    } finally {
+      fs.delete(new Path(remoteLogRootDir), true);
+      fs.delete(new Path(rootLogDir), true);
+    }
+  }
+
+  @Test (timeout = 15000)
   public void testPrintContainerLogMetadata() throws Exception {
     String remoteLogRootDir = "target/logs/";
     Configuration configuration = new Configuration();
@@ -380,7 +524,8 @@ public class TestLogsCLI {
         appId, containerIds, nodeIds);
 
     YarnClient mockYarnClient =
-        createMockYarnClient(YarnApplicationState.FINISHED);
+        createMockYarnClient(YarnApplicationState.FINISHED,
+        UserGroupInformation.getCurrentUser().getShortUserName());
     LogsCLI cli = new LogsCLIForTest(mockYarnClient);
     cli.setConf(configuration);
 
@@ -466,7 +611,8 @@ public class TestLogsCLI {
         appId, containerIds, nodeIds);
 
     YarnClient mockYarnClient =
-        createMockYarnClient(YarnApplicationState.FINISHED);
+        createMockYarnClient(YarnApplicationState.FINISHED,
+        UserGroupInformation.getCurrentUser().getShortUserName());
     LogsCLI cli = new LogsCLIForTest(mockYarnClient);
     cli.setConf(configuration);
 
@@ -508,7 +654,8 @@ public class TestLogsCLI {
     assertTrue(fs.exists(harPath));
 
     YarnClient mockYarnClient =
-        createMockYarnClient(YarnApplicationState.FINISHED);
+        createMockYarnClient(YarnApplicationState.FINISHED,
+            ugi.getShortUserName());
     LogsCLI cli = new LogsCLIForTest(mockYarnClient);
     cli.setConf(configuration);
     int exitCode = cli.run(new String[]{"-applicationId",
@@ -630,10 +777,12 @@ public class TestLogsCLI {
     writer.close();
   }
 
-  private YarnClient createMockYarnClient(YarnApplicationState appState)
+  private YarnClient createMockYarnClient(YarnApplicationState appState,
+      String user)
       throws YarnException, IOException {
     YarnClient mockClient = mock(YarnClient.class);
     ApplicationReport mockAppReport = mock(ApplicationReport.class);
+    doReturn(user).when(mockAppReport).getUser();
     doReturn(appState).when(mockAppReport).getYarnApplicationState();
     doReturn(mockAppReport).when(mockClient).getApplicationReport(
         any(ApplicationId.class));
@@ -659,9 +808,9 @@ public class TestLogsCLI {
   }
 
   private static class LogsCLIForTest extends LogsCLI {
-    
+
     private YarnClient yarnClient;
-    
+
     public LogsCLIForTest(YarnClient yarnClient) {
       super();
       this.yarnClient = yarnClient;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d6a963/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index e3ea1f6..ba0dd89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -23,6 +23,7 @@ import java.io.EOFException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.file.AccessDeniedException;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
@@ -34,6 +35,8 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.HarFs;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
@@ -56,6 +59,53 @@ public class LogCLIHelpers implements Configurable {
 
   @Private
   @VisibleForTesting
+  /**
+   * Return the owner for a given AppId
+   * @param remoteRootLogDir
+   * @param appId
+   * @param bestGuess
+   * @param conf
+   * @return the owner or null
+   * @throws IOException
+   */
+  public static String getOwnerForAppIdOrNull(
+      ApplicationId appId, String bestGuess,
+      Configuration conf) throws IOException {
+    Path remoteRootLogDir = new Path(conf.get(
+        YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+        YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
+    String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
+    Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir,
+        appId, bestGuess, suffix);
+    FileContext fc =
+        FileContext.getFileContext(remoteRootLogDir.toUri(), conf);
+    String pathAccess = fullPath.toString();
+    try {
+      if (fc.util().exists(fullPath)) {
+        return bestGuess;
+      }
+      Path toMatch = LogAggregationUtils.
+          getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix);
+      pathAccess = toMatch.toString();
+      FileStatus[] matching  = fc.util().globStatus(toMatch);
+      if (matching == null || matching.length != 1) {
+        return null;
+      }
+      //fetch the user from the full path /app-logs/user[/suffix]/app_id
+      Path parent = matching[0].getPath().getParent();
+      //skip the suffix too
+      if (suffix != null && !StringUtils.isEmpty(suffix)) {
+        parent = parent.getParent();
+      }
+      return parent.getName();
+    } catch (AccessControlException | AccessDeniedException ex) {
+      logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage());
+      return null;
+    }
+  }
+
+  @Private
+  @VisibleForTesting
   public int dumpAContainersLogsForALogType(String appId, String containerId,
       String nodeId, String jobOwner, List<String> logType)
       throws IOException {
@@ -93,12 +143,12 @@ public class LogCLIHelpers implements Configurable {
                 thisNodeFile.getPath());
           if (logType == null) {
             if (dumpAContainerLogs(containerId, reader, System.out,
-              thisNodeFile.getModificationTime()) > -1) {
+                thisNodeFile.getModificationTime()) > -1) {
               foundContainerLogs = true;
             }
           } else {
             if (dumpAContainerLogsForALogType(containerId, reader, System.out,
-              thisNodeFile.getModificationTime(), logType) > -1) {
+                thisNodeFile.getModificationTime(), logType) > -1) {
               foundContainerLogs = true;
             }
           }
@@ -182,7 +232,7 @@ public class LogCLIHelpers implements Configurable {
     while (true) {
       try {
         LogReader.readAContainerLogsForALogType(valueStream, out,
-          logUploadedTime);
+            logUploadedTime);
         foundContainerLogs = true;
       } catch (EOFException eof) {
         break;
@@ -249,9 +299,10 @@ public class LogCLIHelpers implements Configurable {
         continue;
       }
       if (!thisNodeFile.getPath().getName()
-        .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
+          .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
         AggregatedLogFormat.LogReader reader =
-            new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath());
+            new AggregatedLogFormat.LogReader(getConf(),
+                thisNodeFile.getPath());
         try {
 
           DataInputStream valueStream;
@@ -261,13 +312,14 @@ public class LogCLIHelpers implements Configurable {
           while (valueStream != null) {
 
             String containerString =
-                "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
+                "\n\nContainer: " + key + " on "
+                + thisNodeFile.getPath().getName();
             out.println(containerString);
             out.println(StringUtils.repeat("=", containerString.length()));
             while (true) {
               try {
                 LogReader.readAContainerLogsForALogType(valueStream, out,
-                  thisNodeFile.getModificationTime());
+                    thisNodeFile.getModificationTime());
                 foundAnyLogs = true;
               } catch (EOFException eof) {
                 break;
@@ -283,7 +335,7 @@ public class LogCLIHelpers implements Configurable {
         }
       }
     }
-    if (! foundAnyLogs) {
+    if (!foundAnyLogs) {
       emptyLogDir(getRemoteAppLogDir(appId, appOwner).toString());
       return -1;
     }
@@ -398,6 +450,9 @@ public class LogCLIHelpers implements Configurable {
           getConf()).listStatus(remoteAppLogDir);
     } catch (FileNotFoundException fnf) {
       logDirNotExist(remoteAppLogDir.toString());
+    } catch (AccessControlException | AccessDeniedException ace) {
+      logDirNoAccessPermission(remoteAppLogDir.toString(), appOwner,
+        ace.getMessage());
     }
     return nodeFiles;
   }
@@ -426,7 +481,7 @@ public class LogCLIHelpers implements Configurable {
 
   private static void containerLogNotFound(String containerId) {
     System.err.println("Logs for container " + containerId
-      + " are not present in this log-file.");
+        + " are not present in this log-file.");
   }
 
   private static void logDirNotExist(String remoteAppLogDir) {
@@ -437,4 +492,13 @@ public class LogCLIHelpers implements Configurable {
   private static void emptyLogDir(String remoteAppLogDir) {
     System.err.println(remoteAppLogDir + " does not have any log files.");
   }
+
+  private static void logDirNoAccessPermission(String remoteAppLogDir,
+      String appOwner, String errorMessage) throws IOException {
+    System.err.println("Guessed logs' owner is " + appOwner
+        + " and current user "
+        + UserGroupInformation.getCurrentUser().getUserName() + " does not "
+        + "have permission to access " + remoteAppLogDir
+        + ". Error message found: " + errorMessage);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message