hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ecl...@apache.org
Subject hadoop git commit: MAPREDUCE-6519 Avoid unsafe split and append on fields that might be IPv6 literals
Date Wed, 04 Nov 2015 21:04:28 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-11890 009e67021 -> 9fe251d15


MAPREDUCE-6519 Avoid unsafe split and append on fields that might be IPv6 literals


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fe251d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fe251d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fe251d1

Branch: refs/heads/HADOOP-11890
Commit: 9fe251d154e49b1bf27464465d0523c2880afe6c
Parents: 009e670
Author: Elliott Clark <eclark@apache.org>
Authored: Wed Nov 4 13:01:27 2015 -0800
Committer: Elliott Clark <eclark@apache.org>
Committed: Wed Nov 4 13:01:27 2015 -0800

----------------------------------------------------------------------
 .../apache/hadoop/mapred/FileInputFormat.java   | 23 ++++++++++----------
 .../apache/hadoop/mapreduce/util/HostUtil.java  |  6 ++---
 .../mapreduce/v2/hs/HistoryClientService.java   | 11 ++++++----
 .../hadoop/ipc/TestMRCJCSocketFactory.java      |  8 ++++---
 .../apache/hadoop/mapred/ReliabilityTest.java   | 13 ++---------
 .../hadoop/mapred/TestClientRedirect.java       |  8 ++++---
 .../org/apache/hadoop/mapred/UtilsForTests.java | 16 --------------
 .../mapreduce/MiniHadoopClusterManager.java     |  6 +++--
 8 files changed, 37 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index 2c58ebe..65dbdd2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.mapreduce.security.TokenCache;
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StopWatch;
@@ -685,19 +686,19 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K,
V> {
   
   private String[] identifyHosts(int replicationFactor, 
                                  Map<Node,NodeInfo> racksMap) {
-    
+
     String [] retVal = new String[replicationFactor];
-   
-    List <NodeInfo> rackList = new LinkedList<NodeInfo>(); 
+
+    List <NodeInfo> rackList = new LinkedList<NodeInfo>();
 
     rackList.addAll(racksMap.values());
-    
+
     // Sort the racks based on their contribution to this split
     sortInDescendingOrder(rackList);
-    
+
     boolean done = false;
     int index = 0;
-    
+
     // Get the host list for all our aggregated items, sort
     // them and return the top entries
     for (NodeInfo ni: rackList) {
@@ -706,27 +707,27 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K,
V> {
 
       List<NodeInfo>hostList = new LinkedList<NodeInfo>();
       hostList.addAll(hostSet);
-    
+
       // Sort the hosts in this rack based on their contribution
       sortInDescendingOrder(hostList);
 
       for (NodeInfo host: hostList) {
         // Strip out the port number from the host name
-        retVal[index++] = host.node.getName().split(":")[0];
+        retVal[index++] = NetUtils.getHostFromHostPort(host.node.getName());
         if (index == replicationFactor) {
           done = true;
           break;
         }
       }
-      
+
       if (done == true) {
         break;
       }
     }
     return retVal;
   }
-  
-  private String[] fakeRacks(BlockLocation[] blkLocations, int index) 
+
+  private String[] fakeRacks(BlockLocation[] blkLocations, int index)
   throws IOException {
     String[] allHosts = blkLocations[index].getHosts();
     String[] allTopos = new String[allHosts.length];

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
index ad279ee..1ba4387 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.util;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.net.NetUtils;
 
 @Private
 @Unstable
@@ -56,10 +57,7 @@ public class HostUtil {
   public static String convertTrackerNameToHostName(String trackerName) {
     // Ugly!
     // Convert the trackerName to its host name
-    int indexOfColon = trackerName.indexOf(":");
-    String trackerHostName = (indexOfColon == -1) ? 
-      trackerName : 
-      trackerName.substring(0, indexOfColon);
+    String trackerHostName = NetUtils.getHostFromHostPort(trackerName);
     return trackerHostName.substring("tracker_".length());
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
index 3751ad9..1d5ed5f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.yarn.webapp.WebApp;
 import org.apache.hadoop.yarn.webapp.WebApps;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
 
 /**
  * This module is responsible for talking to the
@@ -161,10 +162,12 @@ public class HistoryClientService extends AbstractService {
         .withHttpSpnegoPrincipalKey(
             JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY)
         .at(NetUtils.getHostPortString(bindAddress)).start(webApp);
-    
-    String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0];
-    MRWebAppUtil.setJHSWebappURLWithoutScheme(conf,
-        connectHost + ":" + webApp.getListenerAddress().getPort());
+
+    String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf);
+
+    MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, HostAndPort.fromParts(
+        HostAndPort.fromString(connectHost).getHostText(),
+        webApp.getListenerAddress().getPort()).toString());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
index 0274298..dc0b09a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestMRCJCSocketFactory.java
@@ -36,6 +36,8 @@ import org.apache.hadoop.net.StandardSocketFactory;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.google.common.net.HostAndPort;
+
 /**
  * This class checks that RPCs can use specialized socket factories.
  */
@@ -88,9 +90,9 @@ public class TestMRCJCSocketFactory {
                 "org.apache.hadoop.ipc.DummySocketFactory");
       jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
       String rmAddress = jconf.get("yarn.resourcemanager.address");
-      String[] split = rmAddress.split(":");
-      jconf.set("yarn.resourcemanager.address", split[0] + ':'
-          + (Integer.parseInt(split[1]) + 10));
+      HostAndPort hp = HostAndPort.fromString(rmAddress);
+      jconf.set("yarn.resourcemanager.address", hp.getHostText() + ':'
+          + (hp.getPort() + 10));
       client = new JobClient(jconf);
 
       JobStatus[] jobs = client.jobsToComplete();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
index e6e12eb..1d1ca2a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.util.HostUtil;
 import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Shell;
@@ -340,7 +341,7 @@ public class ReliabilityTest extends Configured implements Tool {
       LOG.info(new Date() + " Stopping a few trackers");
 
       for (String tracker : trackerNamesList) {
-        String host = convertTrackerNameToHostName(tracker);
+        String host = HostUtil.convertTrackerNameToHostName(tracker);
         LOG.info(new Date() + " Marking tracker on host: " + host);
         fos.write((host + "\n").getBytes());
         if (count++ >= trackerNamesList.size()/2) {
@@ -378,16 +379,6 @@ public class ReliabilityTest extends Configured implements Tool {
         LOG.info(output);
       }
     }
-
-    private String convertTrackerNameToHostName(String trackerName) {
-      // Convert the trackerName to it's host name
-      int indexOfColon = trackerName.indexOf(":");
-      String trackerHostName = (indexOfColon == -1) ? 
-          trackerName : 
-            trackerName.substring(0, indexOfColon);
-      return trackerHostName.substring("tracker_".length());
-    }
-
   }
   
   private class KillTaskThread extends Thread {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
index 8ab3304..51b030c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
@@ -133,6 +133,8 @@ import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.google.common.net.HostAndPort;
+
 public class TestClientRedirect {
 
   static {
@@ -306,9 +308,9 @@ public class TestClientRedirect {
         application.setYarnApplicationState(YarnApplicationState.FINISHED);
         application.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
       }
-      String[] split = AMHOSTADDRESS.split(":");
-      application.setHost(split[0]);
-      application.setRpcPort(Integer.parseInt(split[1]));
+      HostAndPort hp = HostAndPort.fromString(AMHOSTADDRESS);
+      application.setHost(hp.getHostText());
+      application.setRpcPort(hp.getPort());
       application.setUser("TestClientRedirect-user");
       application.setName("N/A");
       application.setQueue("N/A");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
index 972391c..3e7a596 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
@@ -797,20 +797,4 @@ public class UtilsForTests {
     file.close();
     return file;
   }
-
-  /**
-   * This formats the long tasktracker name to just the FQDN
-   * @param taskTrackerLong String The long format of the tasktracker string
-   * @return String The FQDN of the tasktracker
-   * @throws Exception
-   */
-  public static String getFQDNofTT (String taskTrackerLong) throws Exception {
-    //Getting the exact FQDN of the tasktracker from the tasktracker string.
-    String[] firstSplit = taskTrackerLong.split("_");
-    String tmpOutput = firstSplit[1];
-    String[] secondSplit = tmpOutput.split(":");
-    String tmpTaskTracker = secondSplit[0];
-    return tmpTaskTracker;
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fe251d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
index 2e8ba5e..cae01e7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
@@ -47,6 +47,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.mortbay.util.ajax.JSON;
 
+import com.google.common.net.HostAndPort;
+
 /**
  * This class drives the creation of a mini-cluster on the local machine. By
  * default, a MiniDFSCluster and MiniMRCluster are spawned on the first
@@ -188,8 +190,8 @@ public class MiniHadoopClusterManager {
         map.put("namenode_port", dfs.getNameNodePort());
       }
       if (mr != null) {
-        map.put("resourcemanager_port", mr.getConfig().get(
-            YarnConfiguration.RM_ADDRESS).split(":")[1]);
+        map.put("resourcemanager_port", HostAndPort.fromString(
+              mr.getConfig().get(YarnConfiguration.RM_ADDRESS)).getPort());
       }
       FileWriter fw = new FileWriter(new File(writeDetails));
       fw.write(new JSON().toJSON(map));


Mime
View raw message