hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1569524 - in /hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/ma...
Date Tue, 18 Feb 2014 20:42:30 GMT
Author: szetszwo
Date: Tue Feb 18 20:42:26 2014
New Revision: 1569524

URL: http://svn.apache.org/r1569524
Log:
Merge r1555021 through r1569522 from trunk.

Modified:
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/   (props changed)
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt   (contents, props
changed)
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
    hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java

Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1568421-1569522

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt Tue Feb 18 20:42:26
2014
@@ -139,6 +139,18 @@ Trunk (Unreleased)
 
     MAPREDUCE-5717. Task pings are interpreted as task progress (jlowe)
 
+Release 2.5.0 - UNRELEASED
+
+  INCOMPATIBLE CHANGES
+
+  NEW FEATURES
+
+  IMPROVEMENTS
+
+  OPTIMIZATIONS
+
+  BUG FIXES 
+
 Release 2.4.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -165,6 +177,9 @@ Release 2.4.0 - UNRELEASED
     MAPREDUCE-5670. CombineFileRecordReader should report progress when moving
     to the next file (Chen He via jlowe)
 
+    MAPREDUCE-5757. ConcurrentModificationException in JobControl.toList
+    (jlowe)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1568421-1569522

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
Tue Feb 18 20:42:26 2014
@@ -1387,7 +1387,8 @@ public class MRAppMaster extends Composi
       // RM/NM to issue SSL certificates but definitely not MR-AM as it is
       // running in user-land.
       MRWebAppUtil.initialize(conf);
-      HttpConfig.setPolicy(HttpConfig.Policy.HTTP_ONLY);
+      conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
+          HttpConfig.Policy.HTTP_ONLY.name());
       // log the system properties
       String systemPropsToLog = MRApps.getSystemPropertiesToLog(conf);
       if (systemPropsToLog != null) {

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
Tue Feb 18 20:42:26 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.v2.j
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpConfig;
 
 /**
  * Stores Job History configuration keys that can be set by administrators of
@@ -135,7 +136,7 @@ public class JHAdminConfig {
   public static final String MR_HS_HTTP_POLICY = MR_HISTORY_PREFIX
       + "http.policy";
   public static String DEFAULT_MR_HS_HTTP_POLICY =
-          CommonConfigurationKeysPublic.HTTP_POLICY_HTTP_ONLY;
+          HttpConfig.Policy.HTTP_ONLY.name();
   
   /**The address the history server webapp is on.*/
   public static final String MR_HISTORY_WEBAPP_ADDRESS =

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
Tue Feb 18 20:42:26 2014
@@ -71,11 +71,13 @@ public class MRWebAppUtil {
   }
 
   public static String getYARNWebappScheme() {
-    return HttpConfig.getScheme(httpPolicyInYarn);
+    return httpPolicyInYarn == HttpConfig.Policy.HTTPS_ONLY ? "https://"
+        : "http://";
   }
-  
+
   public static String getJHSWebappScheme() {
-    return HttpConfig.getScheme(httpPolicyInJHS);
+    return httpPolicyInJHS == HttpConfig.Policy.HTTPS_ONLY ? "https://"
+        : "http://";
   }
   
   public static void setJHSWebappURLWithoutScheme(Configuration conf,

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
Tue Feb 18 20:42:26 2014
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.jobhi
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
 import org.apache.hadoop.mapreduce.util.HostUtil;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 /**
  * HistoryViewer is used to parse and view the JobHistory files 
@@ -231,7 +232,8 @@ public class HistoryViewer {
           taskList.append("\t"); 
           taskList.append(attempt.getHostname()).append("\t");
           taskList.append(attempt.getError());
-          String taskLogsUrl = getTaskLogsUrl(attempt);
+          String taskLogsUrl = getTaskLogsUrl(
+              WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt);
           taskList.append(taskLogsUrl != null ? taskLogsUrl : "n/a");
           System.out.println(taskList.toString());
         }
@@ -446,7 +448,7 @@ public class HistoryViewer {
    * @return the taskLogsUrl. null if http-port or tracker-name or
    *         task-attempt-id are unavailable.
    */
-  public static String getTaskLogsUrl(
+  public static String getTaskLogsUrl(String scheme,
       JobHistoryParser.TaskAttemptInfo attempt) {
     if (attempt.getHttpPort() == -1
         || attempt.getTrackerName().equals("")
@@ -457,7 +459,7 @@ public class HistoryViewer {
     String taskTrackerName =
       HostUtil.convertTrackerNameToHostName(
         attempt.getTrackerName());
-    return HostUtil.getTaskLogUrl(taskTrackerName,
+    return HostUtil.getTaskLogUrl(scheme, taskTrackerName,
         Integer.toString(attempt.getHttpPort()),
         attempt.getAttemptId().toString());
   }

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java
Tue Feb 18 20:42:26 2014
@@ -79,7 +79,7 @@ public class JobControl implements Runna
     this.runnerState = ThreadState.READY;
   }
 	
-  synchronized private static List<ControlledJob> toList(
+  private static List<ControlledJob> toList(
                    LinkedList<ControlledJob> jobs) {
     ArrayList<ControlledJob> retv = new ArrayList<ControlledJob>();
     for (ControlledJob job : jobs) {
@@ -122,11 +122,11 @@ public class JobControl implements Runna
   /**
    * @return the jobs in the success state
    */
-  public List<ControlledJob> getSuccessfulJobList() {
+  synchronized public List<ControlledJob> getSuccessfulJobList() {
     return toList(this.successfulJobs);
   }
 	
-  public List<ControlledJob> getFailedJobList() {
+  synchronized public List<ControlledJob> getFailedJobList() {
     return toList(this.failedJobs);
   }
 	

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
Tue Feb 18 20:42:26 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.mapreduce.util
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.http.HttpConfig;
 
 @Private
 @Unstable
@@ -33,9 +32,9 @@ public class HostUtil {
    * @param taskAttemptID
    * @return the taskLogUrl
    */
-  public static String getTaskLogUrl(String taskTrackerHostName,
+  public static String getTaskLogUrl(String scheme, String taskTrackerHostName,
     String httpPort, String taskAttemptID) {
-    return (HttpConfig.getSchemePrefix() + taskTrackerHostName + ":" +
+    return (scheme + taskTrackerHostName + ":" +
         httpPort + "/tasklog?attemptid=" + taskAttemptID);
   }
 

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Tue Feb 18 20:42:26 2014
@@ -24,7 +24,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.v2.hs.HistoryServerStateStoreService.HistoryServerState;
@@ -121,7 +120,6 @@ public class JobHistoryServer extends Co
 
     // This is required for WebApps to use https if enabled.
     MRWebAppUtil.initialize(getConfig());
-    HttpConfig.setPolicy(MRWebAppUtil.getJHSHttpPolicy());
     try {
       doSecureLogin(conf);
     } catch(IOException ie) {

Modified: hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1569524&r1=1569523&r2=1569524&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
Tue Feb 18 20:42:26 2014
@@ -230,9 +230,9 @@ public class MiniMRYarnCluster extends M
                WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
       LOG.info("MiniMRYARN HistoryServer address: " +
                getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
-      LOG.info("MiniMRYARN HistoryServer web address: " +
-          getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
-              HttpConfig.isSecure()));
+      LOG.info("MiniMRYARN HistoryServer web address: "
+          + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
+              MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
     }
 
     @Override



Mime
View raw message