tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: Fix build break caused by multiple YARN changes - YARN-840, YARN-841, YARN-834, YARN-610, YARN-822. (sseth)
Date Tue, 18 Jun 2013 07:25:32 GMT
Updated Branches:
  refs/heads/master 911e884ae -> c35702d6f


Fix build break caused by multiple YARN changes -  YARN-840, YARN-841,
YARN-834, YARN-610, YARN-822. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/c35702d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/c35702d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/c35702d6

Branch: refs/heads/master
Commit: c35702d6f9fe8707752189303b9eef0b9e821643
Parents: 911e884
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Jun 18 00:24:56 2013 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Jun 18 00:24:56 2013 -0700

----------------------------------------------------------------------
 .../java/org/apache/tez/client/TezClient.java   |  4 ++--
 .../dag/api/client/rpc/DAGClientRPCImpl.java    |  4 ++--
 .../tez/dag/app/MRClientSecurityInfo.java       |  4 ++--
 .../dag/app/launcher/ContainerLauncherImpl.java |  8 +++----
 .../apache/tez/dag/app/rm/TaskScheduler.java    |  7 +++----
 .../tez/dag/app/rm/TestTaskScheduler.java       | 14 ++++++++++---
 .../common/shuffle/server/ShuffleHandler.java   | 22 ++++++++++++--------
 .../mapreduce/examples/OrderedWordCount.java    |  4 ++--
 .../tez/mapreduce/ResourceMgrDelegate.java      |  6 +++---
 9 files changed, 42 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
index 0b2311f..d4208fa 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -54,8 +54,8 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.YarnClient;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.Apps;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
index fa7319e..75b0098 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.YarnClient;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-dag/src/main/java/org/apache/tez/dag/app/MRClientSecurityInfo.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/MRClientSecurityInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/app/MRClientSecurityInfo.java
index 25c3149..73c9a02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/MRClientSecurityInfo.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/MRClientSecurityInfo.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.security.SecurityInfo;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.TokenInfo;
 import org.apache.hadoop.security.token.TokenSelector;
-import org.apache.hadoop.yarn.security.client.ClientTokenSelector;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
 
 public class MRClientSecurityInfo extends SecurityInfo {
 
@@ -51,7 +51,7 @@ public class MRClientSecurityInfo extends SecurityInfo {
       @Override
       public Class<? extends TokenSelector<? extends TokenIdentifier>>
           value() {
-        return ClientTokenSelector.class;
+        return ClientToAMTokenSelector.class;
       }
     };
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index 9c2bd53..db91622 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
 import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.ProtoUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.app.AppContext;
@@ -163,7 +163,7 @@ public class ContainerLauncherImpl extends AbstractService implements
         startRequest.setContainerLaunchContext(containerLaunchContext);
         StartContainerResponse response = proxy.startContainer(startRequest);
 
-        ByteBuffer portInfo = response.getAllServiceResponse().get(
+        ByteBuffer portInfo = response.getAllServicesMetaData().get(
             ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
         int port = -1;
         if(portInfo != null) {
@@ -362,8 +362,8 @@ public class ContainerLauncherImpl extends AbstractService implements
     // the user in createRemoteUser in this context has to be ContainerID
     UserGroupInformation user = UserGroupInformation
         .createRemoteUser(containerID.toString());
-    org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token = ProtoUtils
-        .convertFromProtoFormat(containerToken, cmAddr);
+    org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token = ConverterUtils
+        .convertFromYarn(containerToken, cmAddr);
     user.addToken(token);
 
     ContainerManagementProtocol proxy = user

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index 04d9d60..4035545 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -42,8 +42,8 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.client.AMRMClient.StoredContainerRequest;
-import org.apache.hadoop.yarn.client.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.AMRMClient.StoredContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.RackResolver;
 import org.apache.tez.dag.api.TezUncheckedException;
@@ -142,8 +142,7 @@ public class TaskScheduler extends AbstractService
                         String appTrackingUrl) {
     super(TaskScheduler.class.getName());
     this.appClient = appClient;
-    this.amRmClient = 
-        new AMRMClientAsync<CookieContainerRequest>(id, 1000, this);
+    this.amRmClient = AMRMClientAsync.createAMRMClientAsync(id, 1000, this);
     this.appHostName = appHostName;
     this.appHostPort = appHostPort;
     this.appTrackingUrl = appTrackingUrl;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index ebc0916..7832aac 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -18,6 +18,16 @@
 
 package org.apache.tez.dag.app.rm;
 
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedList;
@@ -37,7 +47,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.client.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -51,8 +61,6 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
-import static org.mockito.Mockito.*;
-
 public class TestTaskScheduler {
   
   RecordFactory recordFactory =

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java
b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java
index edc43b6..f5dfabb 100644
--- a/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java
+++ b/tez-engine/src/main/java/org/apache/tez/engine/common/shuffle/server/ShuffleHandler.java
@@ -59,9 +59,10 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong;
 import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
+import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
+import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
+import org.apache.hadoop.yarn.server.api.AuxiliaryService;
 import org.apache.tez.common.RunningTaskContext;
 import org.apache.tez.common.TezJobConfig;
 import org.apache.tez.engine.common.security.JobTokenIdentifier;
@@ -101,8 +102,7 @@ import org.jboss.netty.util.CharsetUtil;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
-public class ShuffleHandler extends AbstractService 
-    implements AuxServices.AuxiliaryService {
+public class ShuffleHandler extends AuxiliaryService {
 
   private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
   
@@ -219,13 +219,16 @@ public class ShuffleHandler extends AbstractService
   }
 
   
-  
   @Override
-  public void initApp(String user, ApplicationId appId, ByteBuffer secret) {
+  public void initializeApplication(
+      ApplicationInitializationContext initAppContext) {
     // TODO these bytes should be versioned
     try {
+      String user = initAppContext.getUser();
+      ApplicationId appId = initAppContext.getApplicationId();
+      ByteBuffer secret = initAppContext.getApplicationDataForService();
       Token<JobTokenIdentifier> jt = deserializeServiceData(secret);
-       // TODO: Once SHuffle is out of NM, this can use MR APIs
+      // TODO: Once SHuffle is out of NM, this can use MR APIs
       userRsrc.put(appId.toString(), user);
       LOG.info("Added token for " + appId.toString());
       secretManager.addTokenForJob(appId.toString(), jt);
@@ -236,7 +239,8 @@ public class ShuffleHandler extends AbstractService
   }
 
   @Override
-  public void stopApp(ApplicationId appId) {
+  public void stopApplication(ApplicationTerminationContext context) {
+    ApplicationId appId = context.getApplicationId();
     secretManager.removeTokenForJob(appId.toString());
     userRsrc.remove(appId.toString());
   }
@@ -292,7 +296,7 @@ public class ShuffleHandler extends AbstractService
   }
 
   @Override
-  public synchronized ByteBuffer getMeta() {
+  public synchronized ByteBuffer getMetaData() {
     try {
       return serializeMetaData(port); 
     } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index 42c0c83..78eca75 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -36,8 +36,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.client.YarnClient;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.tez.client.TezClient;
 import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezException;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c35702d6/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
index b14bbde..757675f 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/ResourceMgrDelegate.java
@@ -39,10 +39,10 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ProtoUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
 
 public class ResourceMgrDelegate extends YarnClientImpl {
@@ -105,7 +105,7 @@ public class ResourceMgrDelegate extends YarnClientImpl {
   public Token getDelegationToken(Text renewer) throws IOException,
       InterruptedException {
     try {
-      return ProtoUtils.convertFromProtoFormat(
+      return ConverterUtils.convertFromYarn(
         super.getRMDelegationToken(renewer), rmAddress);
     } catch (YarnException e) {
       throw new IOException(e);


Mime
View raw message