hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1128440 - in /hadoop/mapreduce/branches/MR-279: ./ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/ ...
Date Fri, 27 May 2011 20:13:17 GMT
Author: mahadev
Date: Fri May 27 20:13:17 2011
New Revision: 1128440

URL: http://svn.apache.org/viewvc?rev=1128440&view=rev
Log:
Bugfix for using user staging directory for history files (siddharth seth via mahadev)

Modified:
    hadoop/mapreduce/branches/MR-279/CHANGES.txt
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
    hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
    hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java

Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri May 27 20:13:17 2011
@@ -3,6 +3,8 @@ Hadoop MapReduce Change Log
 Trunk (unreleased changes)
 
   MAPREDUCE-279
+    
+    Bugfix for using user staging directory for history files (siddharth seth via mahadev)
 
     MAPREDUCE-2536. Backporting changes to MR-279.
    

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri May 27 20:13:17 2011
@@ -44,7 +44,6 @@ import org.apache.hadoop.mapreduce.v2.jo
 import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
-import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -102,20 +101,27 @@ public class JobHistoryEventHandler exte
 
     this.conf = conf;
 
-    String logDir = JobHistoryUtils.getConfiguredHistoryLogDirPrefix(conf);
-    String userLogDir = JobHistoryUtils.getHistoryLogDirForUser(conf);
-    String  doneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
-    String userDoneDirPrefix = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+    String logDir = null;
+    String doneDirPrefix = null;
+    String userDoneDirPrefix = null;
+    try {
+      logDir = JobHistoryUtils.getConfiguredHistoryLogDirPrefix(conf);
+      doneDirPrefix = JobHistoryUtils
+          .getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+      userDoneDirPrefix = JobHistoryUtils
+          .getHistoryIntermediateDoneDirForUser(conf);
+    } catch (IOException e) {
+      LOG.error("Failed while getting the configured log directories", e);
+      throw new YarnException(e);
+    }
 
     //Check for the existance of the log dir. Maybe create it. 
-    Path path = null;
     try {
-      path = FileSystem.get(conf).makeQualified(new Path(logDir));
-      logDirFS = FileSystem.get(path.toUri(), conf);
-      LOG.info("Maybe creating staging history logDir: [" + path + "]");
-      mkdir(logDirFS, path, new FsPermission(JobHistoryUtils.HISTORY_STAGING_DIR_PERMISSIONS));
+      logDirPath = FileSystem.get(conf).makeQualified(new Path(logDir));
+      logDirFS = FileSystem.get(logDirPath.toUri(), conf);
+      mkdir(logDirFS, logDirPath, new FsPermission(JobHistoryUtils.HISTORY_STAGING_DIR_PERMISSIONS));
     } catch (IOException e) {
-      LOG.error("Failed while checking for/ceating  history staging path: [" + path + "]",
e);
+      LOG.error("Failed while checking for/ceating  history staging path: [" + logDirPath
+ "]", e);
       throw new YarnException(e);
     }
 
@@ -141,15 +147,6 @@ public class JobHistoryEventHandler exte
       LOG.error("Failed checking for the existance of history intermediate done directory:
[" + doneDirPath + "]");
           throw new YarnException(e);
     }
-        
-    //Check/create staging directory.
-    try {
-      logDirPath = FileSystem.get(conf).makeQualified(new Path(userLogDir));
-      mkdir(logDirFS, logDirPath, new FsPermission(JobHistoryUtils.HISTORY_STAGING_USER_DIR_PERMISSIONS));
-    } catch (IOException e) {
-      LOG.error("Error creating user staging history directory: [" + logDirPath + "]", e);
-      throw new YarnException(e);
-    }
 
     //Check/create user directory under intermediate done dir.
         try {
@@ -267,9 +264,8 @@ public class JobHistoryEventHandler exte
     Configuration conf = getConfig();
 
     long submitTime = (oldFi == null ? context.getClock().getTime() : oldFi.getJobIndexInfo().getSubmitTime());
-
-    // String user = conf.get(MRJobConfig.USER_NAME, System.getProperty("user.name"));
     
+    //TODO Ideally this should be written out to the job dir (.staging/jobid/files - RecoveryService
will need to be patched)
     Path logFile = JobHistoryUtils.getStagingJobHistoryFile(logDirPath, jobId, startCount);
     String user = conf.get(MRJobConfig.USER_NAME);
     if (user == null) {

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
Fri May 27 20:13:17 2011
@@ -151,7 +151,7 @@ public class RecoveryService extends Com
   private void parse() throws IOException {
     // TODO: parse history file based on startCount
     String jobName = TypeConverter.fromYarn(appID).toString();
-    String jobhistoryDir = JobHistoryUtils.getHistoryLogDirForUser(getConfig());
+    String jobhistoryDir = JobHistoryUtils.getConfiguredHistoryLogDirPrefix(getConfig());
     FSDataInputStream in = null;
     Path historyFile = null;
     Path histDirPath = FileContext.getFileContext(getConfig()).makeQualified(

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
Fri May 27 20:13:17 2011
@@ -129,7 +129,7 @@ public class MRApp extends MRAppMaster {
   public Job submit(Configuration conf) throws Exception {
     String user = conf.get(MRJobConfig.USER_NAME, "mapred");
     conf.set(MRJobConfig.USER_NAME, user);
-    conf.set(YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, testAbsPath.toString());
+    conf.set(YARNApplicationConstants.APPS_STAGING_DIR_KEY, testAbsPath.toString());
     conf.setBoolean(JHConfig.CREATE_HISTORY_INTERMEDIATE_BASE_DIR_KEY, true);
 
     init(conf);

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHConfig.java
Fri May 27 20:13:17 2011
@@ -11,10 +11,6 @@ public class JHConfig {
   
   public static final String DEFAULT_HS_BIND_ADDRESS = "0.0.0.0:10020";
 
-  /** Staging Dir for AppMaster **/
-  public static final String HISTORY_STAGING_DIR_KEY =
-       "yarn.historyfile.stagingDir";
-
   /** Done Dir for for AppMaster **/
   public static final String HISTORY_INTERMEDIATE_DONE_DIR_KEY =
        "yarn.historyfile.intermediateDoneDir";

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
Fri May 27 20:13:17 2011
@@ -27,8 +27,6 @@ import java.util.concurrent.atomic.Atomi
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileStatus;
@@ -36,10 +34,11 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
+import org.apache.hadoop.yarn.util.SecurityUtil;
 
 public class JobHistoryUtils {
   
@@ -151,12 +150,10 @@ public class JobHistoryUtils {
    * @param conf
    * @return A string representation of the prefix.
    */
-  public static String getConfiguredHistoryLogDirPrefix(Configuration conf) {
-    String defaultLogDir = conf.get(
-        //TODO Change this to staging directory
-        YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, "/tmp") + "/history/staging";
-    String logDir = conf.get(JHConfig.HISTORY_STAGING_DIR_KEY,
-      defaultLogDir);
+  public static String getConfiguredHistoryLogDirPrefix(Configuration conf) throws IOException
{
+    String user = UserGroupInformation.getCurrentUser().getShortUserName();
+    Path path = SecurityUtil.getStagingAreaDir(conf, user);
+    String logDir = path.toString();
     return logDir;
   }
   
@@ -165,12 +162,12 @@ public class JobHistoryUtils {
    * @param conf
    * @return A string representation of the prefix.
    */
-  public static String getConfiguredHistoryIntermediateDoneDirPrefix(Configuration conf)
{
-    String defaultDoneDir = conf.get(
-        YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, "/tmp") + "/history/done_intermediate";
+  public static String getConfiguredHistoryIntermediateDoneDirPrefix(Configuration conf)
throws IOException {
     String  doneDirPrefix =
-      conf.get(JHConfig.HISTORY_INTERMEDIATE_DONE_DIR_KEY,
-          defaultDoneDir);
+      conf.get(JHConfig.HISTORY_INTERMEDIATE_DONE_DIR_KEY);
+    if (doneDirPrefix == null) {
+      doneDirPrefix = conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY) + "/history/done_intermediate";
+    }
     return doneDirPrefix;
   }
   
@@ -179,33 +176,23 @@ public class JobHistoryUtils {
    * @param conf
    * @return
    */
-  public static String getConfiguredHistoryServerDoneDirPrefix(Configuration conf) {
-    String defaultDoneDir = conf.get(
-        YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, "/tmp") + "/history/done";
+  public static String getConfiguredHistoryServerDoneDirPrefix(Configuration conf) throws
IOException {
     String  doneDirPrefix =
-      conf.get(JHConfig.HISTORY_DONE_DIR_KEY,
-          defaultDoneDir);
-    return getCurrentDoneDir(doneDirPrefix);
+      conf.get(JHConfig.HISTORY_DONE_DIR_KEY);
+    if (doneDirPrefix == null) {
+      doneDirPrefix = conf.get(YARNApplicationConstants.APPS_STAGING_DIR_KEY) + "/history/done";
+    }
+    return doneDirPrefix;
   }
 
   /**
-   * Gets the user directory for In progress history files.
-   * @param conf
-   * @return
-   */
-  public static String getHistoryLogDirForUser(Configuration conf) {
-    return getConfiguredHistoryLogDirPrefix(conf) + File.separator
-        + conf.get(MRJobConfig.USER_NAME);
-  }
-  
-  /**
    * Gets the user directory for intermediate done history files.
    * @param conf
    * @return
    */
-  public static String getHistoryIntermediateDoneDirForUser(Configuration conf) {
+  public static String getHistoryIntermediateDoneDirForUser(Configuration conf) throws IOException
{
     return getConfiguredHistoryIntermediateDoneDirPrefix(conf) + File.separator
-        + conf.get(MRJobConfig.USER_NAME);
+        + UserGroupInformation.getCurrentUser().getShortUserName();
   }
 
   public static boolean shouldCreateNonUserDirectory(Configuration conf) {

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
Fri May 27 20:13:17 2011
@@ -179,9 +179,12 @@ public class JobHistory extends Abstract
     serialNumberFormat = ("%0"
         + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
 
-    
-    String doneDirPrefix = JobHistoryUtils
-        .getConfiguredHistoryServerDoneDirPrefix(conf);
+    String doneDirPrefix = null;
+    try {
+      doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
+    } catch (IOException e) {
+      throw new YarnException(e);
+    }
     try {
       doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(
           new Path(doneDirPrefix));
@@ -191,8 +194,13 @@ public class JobHistory extends Abstract
       throw new YarnException("Error creating done directory: [" + doneDirPrefixPath + "]",
e);
     }
 
-    String intermediateDoneDirPrefix = JobHistoryUtils
-    .getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+    String intermediateDoneDirPrefix = null;
+    try {
+      intermediateDoneDirPrefix = JobHistoryUtils
+      .getConfiguredHistoryIntermediateDoneDirPrefix(conf);
+    } catch (IOException e) {
+      throw new YarnException(e);
+    }
     try {
       intermediateDoneDirPath = FileContext.getFileContext(conf)
           .makeQualified(new Path(intermediateDoneDirPrefix));

Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
(original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
Fri May 27 20:13:17 2011
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.Clien
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHConfig;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -71,9 +72,6 @@ public class MiniMRYarnCluster extends M
     conf.set(YARNApplicationConstants.APPS_STAGING_DIR_KEY, new File(
         getTestWorkDir(),
         "apps_staging_dir/${user.name}/").getAbsolutePath());
-    conf.set(YARNApplicationConstants.APPS_HISTORY_STAGING_DIR_KEY, new File(
-        getTestWorkDir(), "history_staging_dir/${user.name}/")
-        .getAbsolutePath());
     conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
                                              // which shuffle doesn't happen
     //configure the shuffle service in NM

Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java?rev=1128440&r1=1128439&r2=1128440&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java
(original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YARNApplicationConstants.java
Fri May 27 20:13:17 2011
@@ -38,7 +38,7 @@ public class YARNApplicationConstants {
 
   public static final String APPS_STAGING_DIR_KEY = "yarn.apps.stagingDir";
 
-  public static final String APPS_HISTORY_STAGING_DIR_KEY = "yarn.apps.history.stagingDir";
+//  public static final String APPS_HISTORY_STAGING_DIR_KEY = "yarn.apps.history.stagingDir";
   
   public static final String YARN_MAPREDUCE_APP_JAR_PATH =
       "$YARN_HOME/modules/" + HADOOP_MAPREDUCE_CLIENT_APP_JAR_NAME;



Mime
View raw message