Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 20C909142 for ; Wed, 11 Jan 2012 20:54:43 +0000 (UTC) Received: (qmail 33297 invoked by uid 500); 11 Jan 2012 20:54:42 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 33094 invoked by uid 500); 11 Jan 2012 20:54:42 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 33074 invoked by uid 99); 11 Jan 2012 20:54:42 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Jan 2012 20:54:42 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Jan 2012 20:54:38 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B42452388B1B; Wed, 11 Jan 2012 20:54:01 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1230248 [4/5] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/mai... Date: Wed, 11 Jan 2012 20:53:55 -0000 To: mapreduce-commits@hadoop.apache.org From: atm@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120111205401.B42452388B1B@eris.apache.org> Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java Wed Jan 11 20:53:50 2012 @@ -699,7 +699,7 @@ public class TestHsWebServicesAttempts e assertTrue("name not set", (name != null && !name.isEmpty())); JSONArray counters = counterGroup.getJSONArray("counter"); for (int j = 0; j < counters.length(); j++) { - JSONObject counter = counters.getJSONObject(i); + JSONObject counter = counters.getJSONObject(j); String counterName = counter.getString("name"); assertTrue("name not set", (counterName != null && !counterName.isEmpty())); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java Wed Jan 11 20:53:50 2012 @@ -77,7 +77,7 @@ import com.sun.jersey.test.framework.Web * * /ws/v1/history/mapreduce/jobs /ws/v1/history/mapreduce/jobs/{jobid} * /ws/v1/history/mapreduce/jobs/{jobid}/counters - * /ws/v1/history/mapreduce/jobs/{jobid}/attempts + * /ws/v1/history/mapreduce/jobs/{jobid}/jobattempts */ public class TestHsWebServicesJobs extends JerseyTest { @@ -626,12 +626,12 @@ public class TestHsWebServicesJobs exten String jobId = MRApps.toString(id); ClientResponse response = r.path("ws").path("v1").path("history") - .path("mapreduce").path("jobs").path(jobId).path("attempts") + .path("mapreduce").path("jobs").path(jobId).path("jobattempts") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("attempts"); + JSONObject info = json.getJSONObject("jobAttempts"); verifyHsJobAttempts(info, jobsMap.get(id)); } } @@ -644,12 +644,12 @@ public class TestHsWebServicesJobs exten String jobId = MRApps.toString(id); ClientResponse response = r.path("ws").path("v1").path("history") - .path("mapreduce").path("jobs").path(jobId).path("attempts/") + .path("mapreduce").path("jobs").path(jobId).path("jobattempts/") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("attempts"); + JSONObject info = json.getJSONObject("jobAttempts"); verifyHsJobAttempts(info, jobsMap.get(id)); } } @@ -662,12 +662,12 @@ public class TestHsWebServicesJobs exten String jobId = MRApps.toString(id); ClientResponse response = r.path("ws").path("v1").path("history") - .path("mapreduce").path("jobs").path(jobId).path("attempts") + .path("mapreduce").path("jobs").path(jobId).path("jobattempts") .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("attempts"); + JSONObject info = json.getJSONObject("jobAttempts"); verifyHsJobAttempts(info, jobsMap.get(id)); } } @@ -680,7 +680,7 @@ public class TestHsWebServicesJobs exten String jobId = MRApps.toString(id); ClientResponse response = r.path("ws").path("v1").path("history") - .path("mapreduce").path("jobs").path(jobId).path("attempts") + .path("mapreduce").path("jobs").path(jobId).path("jobattempts") .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); String xml = response.getEntity(String.class); @@ -689,9 +689,9 @@ public class TestHsWebServicesJobs exten InputSource is = new InputSource(); is.setCharacterStream(new StringReader(xml)); Document dom = db.parse(is); - NodeList attempts = dom.getElementsByTagName("attempts"); + NodeList attempts = dom.getElementsByTagName("jobAttempts"); assertEquals("incorrect number of elements", 1, attempts.getLength()); - NodeList info = dom.getElementsByTagName("attempt"); + NodeList info = dom.getElementsByTagName("jobAttempt"); verifyHsJobAttemptsXML(info, jobsMap.get(id)); } } @@ -699,7 +699,7 @@ public class TestHsWebServicesJobs exten public void verifyHsJobAttempts(JSONObject info, Job job) throws JSONException { - JSONArray attempts = info.getJSONArray("attempt"); + JSONArray attempts = info.getJSONArray("jobAttempt"); assertEquals("incorrect number of elements", 2, attempts.length()); for (int i = 0; i < attempts.length(); i++) { JSONObject attempt = attempts.getJSONObject(i); @@ -732,9 +732,10 @@ public class TestHsWebServicesJobs exten if (amInfo.getAppAttemptId().getAttemptId() == id) { attemptFound = true; String nmHost = amInfo.getNodeManagerHost(); - int nmPort = amInfo.getNodeManagerHttpPort(); + int nmHttpPort = amInfo.getNodeManagerHttpPort(); + int nmPort = amInfo.getNodeManagerPort(); WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":" - + nmPort, nodeHttpAddress); + + nmHttpPort, nodeHttpAddress); WebServicesTestUtils.checkStringMatch("nodeId", BuilderUtils.newNodeId(nmHost, nmPort).toString(), nodeId); assertTrue("startime not greater than 0", startTime > 0); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java Wed Jan 11 20:53:50 2012 @@ -788,7 +788,7 @@ public class TestHsWebServicesTasks exte assertTrue("name not set", (name != null && !name.isEmpty())); JSONArray counters = counterGroup.getJSONArray("counter"); for (int j = 0; j < counters.length(); j++) { - JSONObject counter = counters.getJSONObject(i); + JSONObject counter = counters.getJSONObject(j); String counterName = counter.getString("name"); assertTrue("name not set", (counterName != null && !counterName.isEmpty())); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java Wed Jan 11 20:53:50 2012 @@ -70,7 +70,15 @@ public class ClientCache { return client; } - private MRClientProtocol instantiateHistoryProxy() + protected synchronized MRClientProtocol getInitializedHSProxy() + throws IOException { + if (this.hsProxy == null) { + hsProxy = instantiateHistoryProxy(); + } + return this.hsProxy; + } + + protected MRClientProtocol instantiateHistoryProxy() throws IOException { final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); if (StringUtils.isEmpty(serviceAddr)) { Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Wed Jan 11 20:53:50 2012 @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobStatus; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.TypeConverter; @@ -68,7 +69,6 @@ import org.apache.hadoop.yarn.YarnExcept import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -94,6 +94,8 @@ public class ClientServiceDelegate { private static String UNKNOWN_USER = "Unknown User"; private String trackingUrl; + private boolean amAclDisabledStatusLogged = false; + public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm, JobID jobId, MRClientProtocol historyServerProxy) { this.conf = new Configuration(conf); // Cloning for modifying. @@ -157,7 +159,7 @@ public class ClientServiceDelegate { application = rm.getApplicationReport(appId); continue; } - if(!conf.getBoolean(YarnConfiguration.RM_AM_NETWORK_ACL_CLOSED, false)) { + if(!conf.getBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, false)) { UserGroupInformation newUgi = UserGroupInformation.createRemoteUser( UserGroupInformation.getCurrentUser().getUserName()); serviceAddr = application.getHost() + ":" + application.getRpcPort(); @@ -182,12 +184,14 @@ public class ClientServiceDelegate { return instantiateAMProxy(tempStr); } }); - } else { - logApplicationReportInfo(application); - LOG.info("Network ACL closed to AM for job " + jobId - + ". Redirecting to job history server."); - return checkAndGetHSProxy(null, JobState.RUNNING); - } + } else { + if (!amAclDisabledStatusLogged) { + LOG.info("Network ACL closed to AM for job " + jobId + + ". Not going to try to reach the AM."); + amAclDisabledStatusLogged = true; + } + return getNotRunningJob(null, JobState.RUNNING); + } return realProxy; } catch (IOException e) { //possibly the AM has crashed @@ -248,55 +252,10 @@ public class ClientServiceDelegate { return realProxy; } - private void logApplicationReportInfo(ApplicationReport application) { - if(application == null) { - return; - } - LOG.info("AppId: " + application.getApplicationId() - + " # reserved containers: " - + application.getApplicationResourceUsageReport().getNumReservedContainers() - + " # used containers: " - + application.getApplicationResourceUsageReport().getNumUsedContainers() - + " Needed resources (memory): " - + application.getApplicationResourceUsageReport().getNeededResources().getMemory() - + " Reserved resources (memory): " - + application.getApplicationResourceUsageReport().getReservedResources().getMemory() - + " Used resources (memory): " - + application.getApplicationResourceUsageReport().getUsedResources().getMemory() - + " Diagnostics: " - + application.getDiagnostics() - + " Start time: " - + application.getStartTime() - + " Finish time: " - + application.getFinishTime() - + " Host: " - + application.getHost() - + " Name: " - + application.getName() - + " Orig. tracking url: " - + application.getOriginalTrackingUrl() - + " Queue: " - + application.getQueue() - + " RPC port: " - + application.getRpcPort() - + " Tracking url: " - + application.getTrackingUrl() - + " User: " - + application.getUser() - + " Client token: " - + application.getClientToken() - + " Final appl. status: " - + application.getFinalApplicationStatus() - + " Yarn appl. state: " - + application.getYarnApplicationState() - ); - } - private MRClientProtocol checkAndGetHSProxy( ApplicationReport applicationReport, JobState state) { if (null == historyServerProxy) { - LOG.warn("Job History Server is not configured or " + - "job information not yet available on History Server."); + LOG.warn("Job History Server is not configured."); return getNotRunningJob(applicationReport, state); } return historyServerProxy; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java Wed Jan 11 20:53:50 2012 @@ -27,6 +27,8 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest; @@ -201,4 +203,10 @@ public class NotRunningJob implements MR return resp; } + @Override + public GetDelegationTokenResponse getDelegationToken( + GetDelegationTokenRequest request) throws YarnRemoteException { + /* Should not be invoked by anyone. */ + throw new NotImplementedException(); + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Wed Jan 11 20:53:50 2012 @@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.Queue import org.apache.hadoop.mapreduce.TaskTrackerInfo; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -59,12 +61,14 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; +import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; // TODO: This should be part of something like yarn-client. @@ -156,11 +160,20 @@ public class ResourceMgrDelegate { } - public Token getDelegationToken(Text arg0) + @SuppressWarnings("rawtypes") + public Token getDelegationToken(Text renewer) throws IOException, InterruptedException { - // TODO: Implement getDelegationToken - LOG.warn("getDelegationToken - Not Implemented"); - return null; + /* get the token from RM */ + org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest + rmDTRequest = recordFactory.newRecordInstance( + org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest.class); + rmDTRequest.setRenewer(renewer.toString()); + org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse + response = applicationsManager.getDelegationToken(rmDTRequest); + DelegationToken yarnToken = response.getRMDelegationToken(); + return new Token(yarnToken.getIdentifier().array(), + yarnToken.getPassword().array(), + new Text(yarnToken.getKind()), new Text(yarnToken.getService())); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Wed Jan 11 20:53:50 2012 @@ -28,6 +28,7 @@ import java.util.Vector; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; @@ -54,6 +55,9 @@ import org.apache.hadoop.mapreduce.TypeC import org.apache.hadoop.mapreduce.protocol.ClientProtocol; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.mapreduce.v2.LogParams; +import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.Credentials; @@ -68,6 +72,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -84,6 +89,7 @@ import org.apache.hadoop.yarn.util.Conve /** * This class enables the current JobClient (0.22 hadoop) to run on YARN. */ +@SuppressWarnings({ "rawtypes", "unchecked", "deprecation" }) public class YARNRunner implements ClientProtocol { private static final Log LOG = LogFactory.getLog(YARNRunner.class); @@ -93,7 +99,15 @@ public class YARNRunner implements Clien private ClientCache clientCache; private Configuration conf; private final FileContext defaultFileContext; - + + /* usually is false unless the jobclient getdelegation token is + * called. This is a hack wherein we do return a token from RM + * on getDelegationtoken but due to the restricted api on jobclient + * we just add a job history DT token when submitting a job. + */ + private static final boolean DEFAULT_HS_DELEGATION_TOKEN_REQUIRED = + false; + /** * Yarn runner incapsulates the client interface of * yarn @@ -131,7 +145,16 @@ public class YARNRunner implements Clien throw new RuntimeException("Error in instantiating YarnClient", ufe); } } - + + @Private + /** + * Used for testing mostly. + * @param resMgrDelegate the resource manager delegate to set to. + */ + public void setResourceMgrDelegate(ResourceMgrDelegate resMgrDelegate) { + this.resMgrDelegate = resMgrDelegate; + } + @Override public void cancelDelegationToken(Token arg0) throws IOException, InterruptedException { @@ -161,10 +184,26 @@ public class YARNRunner implements Clien return resMgrDelegate.getClusterMetrics(); } + private Token getDelegationTokenFromHS( + MRClientProtocol hsProxy, Text renewer) throws IOException, + InterruptedException { + GetDelegationTokenRequest request = recordFactory + .newRecordInstance(GetDelegationTokenRequest.class); + request.setRenewer(renewer.toString()); + DelegationToken mrDelegationToken = hsProxy.getDelegationToken(request) + .getDelegationToken(); + return new Token(mrDelegationToken + .getIdentifier().array(), mrDelegationToken.getPassword().array(), + new Text(mrDelegationToken.getKind()), new Text( + mrDelegationToken.getService())); + } + @Override - public Token getDelegationToken(Text arg0) + public Token getDelegationToken(Text renewer) throws IOException, InterruptedException { - return resMgrDelegate.getDelegationToken(arg0); + // The token is only used for serialization. So the type information + // mismatch should be fine. + return resMgrDelegate.getDelegationToken(renewer); } @Override @@ -224,7 +263,16 @@ public class YARNRunner implements Clien @Override public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException, InterruptedException { - + // JobClient will set this flag if getDelegationToken is called, if so, get + // the delegation tokens for the HistoryServer also. + if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED, + DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) { + Token hsDT = getDelegationTokenFromHS(clientCache. + getInitializedHSProxy(), new Text( + conf.get(JobClient.HS_DELEGATION_TOKEN_RENEWER))); + ts.addToken(hsDT.getService(), hsDT); + } + // Upload only in security mode: TODO Path applicationTokensFile = new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Wed Jan 11 20:53:50 2012 @@ -76,6 +76,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -330,6 +332,12 @@ public class TestClientRedirect { GetQueueUserAclsInfoRequest request) throws YarnRemoteException { return null; } + + @Override + public GetDelegationTokenResponse getDelegationToken( + GetDelegationTokenRequest request) throws YarnRemoteException { + return null; + } } class HistoryService extends AMService { @@ -482,6 +490,13 @@ public class TestClientRedirect { FailTaskAttemptRequest request) throws YarnRemoteException { return null; } + + @Override + public org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse getDelegationToken( + org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest request) + throws YarnRemoteException { + return null; + } } static Counters getMyCounters() { Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Wed Jan 11 20:53:50 2012 @@ -19,7 +19,12 @@ package org.apache.hadoop.mapred; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.IOException; import java.util.Arrays; @@ -31,6 +36,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -165,7 +171,7 @@ public class TestClientServiceDelegate { ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate( historyServerProxy, rm); - JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); + JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); Assert.assertNotNull(jobStatus); Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile()); Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl()); @@ -254,6 +260,58 @@ public class TestClientServiceDelegate { any(String.class)); } + @Test + public void testAMAccessDisabled() throws IOException { + //test only applicable when AM not reachable + if(isAMReachableFromClient) { + return; + } + + MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); + when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn( + getJobReportResponseFromHistoryServer()); + + ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class); + when(rmDelegate.getApplicationReport(jobId.getAppId())).thenReturn( + getRunningApplicationReport("am1", 78)).thenReturn( + getRunningApplicationReport("am1", 78)).thenReturn( + getRunningApplicationReport("am1", 78)).thenReturn( + getFinishedApplicationReport()); + + ClientServiceDelegate clientServiceDelegate = spy(getClientServiceDelegate( + historyServerProxy, rmDelegate)); + + JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); + Assert.assertNotNull(jobStatus); + Assert.assertEquals("N/A", jobStatus.getJobName()); + + verify(clientServiceDelegate, times(0)).instantiateAMProxy( + any(String.class)); + + // Should not reach AM even for second and third times too. + jobStatus = clientServiceDelegate.getJobStatus(oldJobId); + Assert.assertNotNull(jobStatus); + Assert.assertEquals("N/A", jobStatus.getJobName()); + verify(clientServiceDelegate, times(0)).instantiateAMProxy( + any(String.class)); + jobStatus = clientServiceDelegate.getJobStatus(oldJobId); + Assert.assertNotNull(jobStatus); + Assert.assertEquals("N/A", jobStatus.getJobName()); + verify(clientServiceDelegate, times(0)).instantiateAMProxy( + any(String.class)); + + // The third time around, app is completed, so should go to JHS + JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId); + Assert.assertNotNull(jobStatus1); + Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile()); + Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl()); + Assert.assertEquals(1.0f, jobStatus1.getMapProgress()); + Assert.assertEquals(1.0f, jobStatus1.getReduceProgress()); + + verify(clientServiceDelegate, times(0)).instantiateAMProxy( + any(String.class)); + } + private GetJobReportRequest getJobReportRequest() { GetJobReportRequest request = Records.newRecord(GetJobReportRequest.class); request.setJobId(jobId); @@ -300,7 +358,7 @@ public class TestClientServiceDelegate { MRClientProtocol historyServerProxy, ResourceMgrDelegate rm) { Configuration conf = new YarnConfiguration(); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - conf.setBoolean(YarnConfiguration.RM_AM_NETWORK_ACL_CLOSED, !isAMReachableFromClient); + conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate( conf, rm, oldJobId, historyServerProxy); return clientServiceDelegate; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java Wed Jan 11 20:53:50 2012 @@ -18,18 +18,34 @@ package org.apache.hadoop.mapreduce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.when; import java.io.IOException; +import java.nio.ByteBuffer; import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.ResourceMgrDelegate; import org.apache.hadoop.mapred.YARNRunner; import org.apache.hadoop.mapreduce.protocol.ClientProtocol; -import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; +import org.apache.hadoop.yarn.api.records.DelegationToken; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.junit.Test; public class TestYarnClientProtocolProvider extends TestCase { - + + private static final RecordFactory recordFactory = RecordFactoryProvider. + getRecordFactory(null); + @Test public void testClusterWithYarnClientProtocolProvider() throws Exception { @@ -68,7 +84,24 @@ public class TestYarnClientProtocolProvi conf = new Configuration(); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); cluster = new Cluster(conf); - cluster.getDelegationToken(new Text(" ")); + YARNRunner yrunner = (YARNRunner) cluster.getClient(); + GetDelegationTokenResponse getDTResponse = + recordFactory.newRecordInstance(GetDelegationTokenResponse.class); + DelegationToken rmDTToken = recordFactory.newRecordInstance( + DelegationToken.class); + rmDTToken.setIdentifier(ByteBuffer.wrap(new byte[2])); + rmDTToken.setKind("Testclusterkind"); + rmDTToken.setPassword(ByteBuffer.wrap("testcluster".getBytes())); + rmDTToken.setService("0.0.0.0:8040"); + getDTResponse.setRMDelegationToken(rmDTToken); + ClientRMProtocol cRMProtocol = mock(ClientRMProtocol.class); + when(cRMProtocol.getDelegationToken(any( + GetDelegationTokenRequest.class))).thenReturn(getDTResponse); + ResourceMgrDelegate rmgrDelegate = new ResourceMgrDelegate( + new YarnConfiguration(conf), cRMProtocol); + yrunner.setResourceMgrDelegate(rmgrDelegate); + Token t = cluster.getDelegationToken(new Text(" ")); + assertTrue("Testclusterkind".equals(t.getKind().toString())); } finally { if (cluster != null) { cluster.close(); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Wed Jan 11 20:53:50 2012 @@ -164,7 +164,7 @@ public class ShuffleHandler extends Abst * @param port the port to be sent to the ApplciationMaster * @return the serialized form of the port. */ - static ByteBuffer serializeMetaData(int port) throws IOException { + public static ByteBuffer serializeMetaData(int port) throws IOException { //TODO these bytes should be versioned DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(port); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java Wed Jan 11 20:53:50 2012 @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.api; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; @@ -43,6 +45,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; @@ -230,4 +233,19 @@ public interface ClientRMProtocol { public GetQueueUserAclsInfoResponse getQueueUserAcls( GetQueueUserAclsInfoRequest request) throws YarnRemoteException; + + /** + *

The interface used by clients to get delegation token, enabling the + * containers to be able to talk to the service using those tokens. + * + *

The ResourceManager responds with the delegation token + * {@link DelegationToken} that can be used by the client to speak to this + * service. + * @param request request to get a delegation token for the client. + * @return delegation token that can be used to talk to this service + * @throws YarnRemoteException + */ + public GetDelegationTokenResponse getDelegationToken( + GetDelegationTokenRequest request) + throws YarnRemoteException; } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/client_RM_protocol.proto Wed Jan 11 20:53:50 2012 @@ -33,5 +33,6 @@ service ClientRMProtocolService { rpc getClusterNodes (GetClusterNodesRequestProto) returns (GetClusterNodesResponseProto); rpc getQueueInfo (GetQueueInfoRequestProto) returns (GetQueueInfoResponseProto); rpc getQueueUserAcls (GetQueueUserAclsInfoRequestProto) returns (GetQueueUserAclsInfoResponseProto); + rpc getDelegationToken(GetDelegationTokenRequestProto) returns (GetDelegationTokenResponseProto); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Wed Jan 11 20:53:50 2012 @@ -65,6 +65,13 @@ message ContainerTokenProto { optional string service = 4; } +message DelegationTokenProto { + optional bytes identifier = 1; + optional bytes password = 2; + optional string kind = 3; + optional string service = 4; +} + message ContainerProto { optional ContainerIdProto id = 1; optional NodeIdProto nodeId = 2; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Wed Jan 11 20:53:50 2012 @@ -138,6 +138,15 @@ message GetQueueUserAclsInfoResponseProt repeated QueueUserACLInfoProto queueUserAcls = 1; } + +message GetDelegationTokenRequestProto { + optional string renewer = 1; +} + +message GetDelegationTokenResponseProto { + optional DelegationTokenProto application_token = 1; +} + ////////////////////////////////////////////////////// /////// client_NM_Protocol /////////////////////////// ////////////////////////////////////////////////////// Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ClientRMProtocolPBClientImpl.java Wed Jan 11 20:53:50 2012 @@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -51,6 +53,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetDelegationTokenRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetDelegationTokenResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl; @@ -68,6 +72,7 @@ import org.apache.hadoop.yarn.proto.Yarn import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto; @@ -247,4 +252,22 @@ public class ClientRMProtocolPBClientImp } } + @Override + public GetDelegationTokenResponse getDelegationToken( + GetDelegationTokenRequest request) throws YarnRemoteException { + GetDelegationTokenRequestProto requestProto = + ((GetDelegationTokenRequestPBImpl)request).getProto(); + try { + return new GetDelegationTokenResponsePBImpl( + proxy.getDelegationToken(null, requestProto)); + } catch (ServiceException e) { + if (e.getCause() instanceof YarnRemoteException) { + throw (YarnRemoteException)e.getCause(); + } else if (e.getCause() instanceof UndeclaredThrowableException) { + throw (UndeclaredThrowableException)e.getCause(); + } else { + throw new UndeclaredThrowableException(e); + } + } + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ClientRMProtocolPBServiceImpl.java Wed Jan 11 20:53:50 2012 @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; @@ -36,6 +37,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetDelegationTokenRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetDelegationTokenResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetNewApplicationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetQueueInfoRequestPBImpl; @@ -56,6 +59,8 @@ import org.apache.hadoop.yarn.proto.Yarn import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterMetricsResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetDelegationTokenResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNewApplicationResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; @@ -194,4 +199,17 @@ public class ClientRMProtocolPBServiceIm } } + @Override + public GetDelegationTokenResponseProto getDelegationToken( + RpcController controller, GetDelegationTokenRequestProto proto) + throws ServiceException { + GetDelegationTokenRequestPBImpl request = + new GetDelegationTokenRequestPBImpl(proto); + try { + GetDelegationTokenResponse response = real.getDelegationToken(request); + return ((GetDelegationTokenResponsePBImpl)response).getProto(); + } catch (YarnRemoteException e) { + throw new ServiceException(e); + } + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Jan 11 20:53:50 2012 @@ -145,11 +145,6 @@ public class YarnConfiguration extends C /** ACL used in case none is found. Allows nothing. */ public static final String DEFAULT_YARN_APP_ACL = " "; - /** RM-AM ACL disabled. **/ - public static final String RM_AM_NETWORK_ACL_CLOSED = - RM_PREFIX + "am.acl.disabled"; - public static final boolean DEFAULT_RM_AM_NETWORK_ACL_CLOSED = false; - /** The address of the RM admin interface.*/ public static final String RM_ADMIN_ADDRESS = RM_PREFIX + "admin.address"; @@ -194,10 +189,26 @@ public class YarnConfiguration extends C /** The class to use as the resource scheduler.*/ public static final String RM_SCHEDULER = RM_PREFIX + "scheduler.class"; + + + //Delegation token related keys + public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = + RM_PREFIX + "delegation.key.update-interval"; + public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = + 24*60*60*1000; // 1 day + public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = + RM_PREFIX + "delegation.token.renew-interval"; + public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = + 24*60*60*1000; // 1 day + public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = + RM_PREFIX + "delegation.token.max-lifetime"; + public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = + 7*24*60*60*1000; // 7 days + /** The class to use as the persistent store.*/ public static final String RM_STORE = RM_PREFIX + "store.class"; - + /** The address of the zookeeper instance to use with ZK store.*/ public static final String RM_ZK_STORE_ADDRESS = RM_PREFIX + "zookeeper-store.address"; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientRMSecurityInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientRMSecurityInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientRMSecurityInfo.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientRMSecurityInfo.java Wed Jan 11 20:53:50 2012 @@ -23,7 +23,9 @@ import java.lang.annotation.Annotation; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.KerberosInfo; import org.apache.hadoop.security.SecurityInfo; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.TokenInfo; +import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.ClientRMProtocol; @@ -56,7 +58,22 @@ public class ClientRMSecurityInfo extend @Override public TokenInfo getTokenInfo(Class protocol, Configuration conf) { - return null; - } + if (!protocol + .equals(ClientRMProtocol.ClientRMProtocolService.BlockingInterface.class)) { + return null; + } + return new TokenInfo() { + + @Override + public Class annotationType() { + return null; + } + @Override + public Class> + value() { + return RMTokenSelector.class; + } + }; + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Wed Jan 11 20:53:50 2012 @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.DelegationToken; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.URL; @@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; /** * Builder utilities to construct various objects. @@ -221,6 +223,18 @@ public class BuilderUtils { return container; } + public static DelegationToken newDelegationToken( + byte[] identifier, String kind, byte[] password, + String service) { + DelegationToken delegationToken = recordFactory.newRecordInstance( + DelegationToken.class); + delegationToken.setIdentifier(ByteBuffer.wrap(identifier)); + delegationToken.setKind(kind); + delegationToken.setPassword(ByteBuffer.wrap(password)); + delegationToken.setService(service); + return delegationToken; + } + public static ContainerToken newContainerToken(NodeId nodeId, ByteBuffer password, ContainerTokenIdentifier tokenIdentifier) { ContainerToken containerToken = recordFactory Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml Wed Jan 11 20:53:50 2012 @@ -117,12 +117,6 @@ - Network ACL to AM closed. - yarn.resourcemanager.am.acl.disabled - false - - - The address of the RM admin interface. yarn.resourcemanager.admin.address 0.0.0.0:8141 Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Wed Jan 11 20:53:50 2012 @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager; @@ -28,12 +28,15 @@ import java.util.concurrent.atomic.Atomi import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.Server; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.authorize.PolicyProvider; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ClientRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; @@ -43,6 +46,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -53,8 +58,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -67,6 +72,8 @@ import org.apache.hadoop.yarn.factories. import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -78,6 +85,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.service.AbstractService; +import org.apache.hadoop.yarn.util.BuilderUtils; /** @@ -97,18 +105,22 @@ public class ClientRMService extends Abs private String clientServiceBindAddress; private Server server; + private RMDelegationTokenSecretManager rmDTSecretManager; + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); InetSocketAddress clientBindAddress; private final ApplicationACLsManager applicationsACLsManager; - + public ClientRMService(RMContext rmContext, YarnScheduler scheduler, - RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager) { + RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, + RMDelegationTokenSecretManager rmDTSecretManager) { super(ClientRMService.class.getName()); this.scheduler = scheduler; this.rmContext = rmContext; this.rmAppManager = rmAppManager; this.applicationsACLsManager = applicationACLsManager; + this.rmDTSecretManager = rmDTSecretManager; } @Override @@ -118,21 +130,19 @@ public class ClientRMService extends Abs YarnConfiguration.DEFAULT_RM_ADDRESS); clientBindAddress = NetUtils.createSocketAddr(clientServiceBindAddress, - YarnConfiguration.DEFAULT_RM_PORT, - YarnConfiguration.RM_ADDRESS); + YarnConfiguration.DEFAULT_RM_PORT, + YarnConfiguration.RM_ADDRESS); super.init(conf); } @Override public void start() { - // All the clients to appsManager are supposed to be authenticated via - // Kerberos if security is enabled, so no secretManager. Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); this.server = rpc.getServer(ClientRMProtocol.class, this, clientBindAddress, - conf, null, + conf, this.rmDTSecretManager, conf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT)); @@ -423,6 +433,49 @@ public class ClientRMService extends Abs return response; } + + @Override + public GetDelegationTokenResponse getDelegationToken( + GetDelegationTokenRequest request) throws YarnRemoteException { + try { + + // Verify that the connection is kerberos authenticated + AuthenticationMethod authMethod = UserGroupInformation + .getRealAuthenticationMethod(UserGroupInformation.getCurrentUser()); + if (UserGroupInformation.isSecurityEnabled() + && (authMethod != AuthenticationMethod.KERBEROS)) { + throw new IOException( + "Delegation Token can be issued only with kerberos authentication"); + } + + GetDelegationTokenResponse response = + recordFactory.newRecordInstance(GetDelegationTokenResponse.class); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + Text owner = new Text(ugi.getUserName()); + Text realUser = null; + if (ugi.getRealUser() != null) { + realUser = new Text(ugi.getRealUser().getUserName()); + } + RMDelegationTokenIdentifier tokenIdentifier = + new RMDelegationTokenIdentifier(owner, new Text(request.getRenewer()), + realUser); + Token realRMDTtoken = + new Token(tokenIdentifier, + this.rmDTSecretManager); + response.setRMDelegationToken( + BuilderUtils.newDelegationToken( + realRMDTtoken.getIdentifier(), + realRMDTtoken.getKind().toString(), + realRMDTtoken.getPassword(), + clientBindAddress.getAddress().getHostAddress() + ":" + + clientBindAddress.getPort() + )); + return response; + } catch(IOException io) { + throw RPCUtil.getRemoteException(io); + } + } + void refreshServiceAcls(Configuration configuration, PolicyProvider policyProvider) { this.server.refreshServiceAcl(configuration, policyProvider); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Wed Jan 11 20:53:50 2012 @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager; @@ -41,12 +41,13 @@ import org.apache.hadoop.yarn.event.Disp import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; +import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; @@ -66,16 +67,16 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; -import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher; -import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; -import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; -import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.service.CompositeService; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.WebApps.Builder; +import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher; +import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet; +import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; /** * The ResourceManager is the main class that is a set of components. @@ -107,7 +108,7 @@ public class ResourceManager extends Com private EventHandler schedulerDispatcher; protected RMAppManager rmAppManager; protected ApplicationACLsManager applicationACLsManager; - + protected RMDelegationTokenSecretManager rmDTSecretManager; private WebApp webApp; private RMContext rmContext; private final Store store; @@ -193,7 +194,7 @@ public class ResourceManager extends Com // Register event handler for RMAppManagerEvents this.rmDispatcher.register(RMAppManagerEventType.class, this.rmAppManager); - + this.rmDTSecretManager = createRMDelegationTokenSecretManager(); clientRM = createClientRMService(); addService(clientRM); @@ -435,7 +436,12 @@ public class ResourceManager extends Com startWepApp(); DefaultMetricsSystem.initialize("ResourceManager"); - + try { + rmDTSecretManager.startThreads(); + } catch(IOException ie) { + throw new YarnException("Failed to start secret manager threads", ie); + } + super.start(); /*synchronized(shutdown) { @@ -459,6 +465,7 @@ public class ResourceManager extends Com if (webApp != null) { webApp.stop(); } + rmDTSecretManager.stopThreads(); /*synchronized(shutdown) { shutdown.set(true); @@ -475,9 +482,25 @@ public class ResourceManager extends Com this.nmLivelinessMonitor, this.containerTokenSecretManager); } + protected RMDelegationTokenSecretManager + createRMDelegationTokenSecretManager() { + long secretKeyInterval = + conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY, + YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); + long tokenMaxLifetime = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + long tokenRenewInterval = + conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, + YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + + return new RMDelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, 3600000); + } + protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, - this.applicationACLsManager); + this.applicationACLsManager, this.rmDTSecretManager); } protected ApplicationMasterService createApplicationMasterService() { Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Wed Jan 11 20:53:50 2012 @@ -178,7 +178,7 @@ public class MockRM extends ResourceMana @Override protected ClientRMService createClientRMService() { return new ClientRMService(getRMContext(), getResourceScheduler(), - rmAppManager, applicationACLsManager) { + rmAppManager, applicationACLsManager, null) { @Override public void start() { // override to not start rpc handler Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Wed Jan 11 20:53:50 2012 @@ -89,7 +89,7 @@ public class TestApplicationACLs { resourceManager = new MockRM(conf) { protected ClientRMService createClientRMService() { return new ClientRMService(getRMContext(), this.scheduler, - this.rmAppManager, this.applicationACLsManager); + this.rmAppManager, this.applicationACLsManager, null); }; }; new Thread() { Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/c++:1159757-1230204 /hadoop/core/branches/branch-0.19/mapred/src/c++:713112 /hadoop/core/trunk/src/c++:776175-784663 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:1152502-1230204 /hadoop/core/branches/branch-0.19/mapred/src/contrib:713112 /hadoop/core/trunk/src/contrib:784664-785643 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:1152502-1230204 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/block_forensics:713112 /hadoop/core/branches/branch-0.19/mapred/src/contrib/block_forensics:713112 /hadoop/core/trunk/src/contrib/block_forensics:784664-785643 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:1161333-1230204 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112 /hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:1161333-1230204 /hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112 /hadoop/core/trunk/src/contrib/build.xml:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:1159757-1230204 /hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112 /hadoop/core/trunk/src/contrib/data_join:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,4 +1,4 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:1159757-1230204 /hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112 /hadoop/core/branches/branch-0.19/mapred/src/contrib/eclipse-plugin:713112 /hadoop/core/trunk/src/contrib/eclipse-plugin:776175-785643 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:1159757-1230204 /hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112 /hadoop/core/trunk/src/contrib/index:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:1159757-1230204 /hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112 /hadoop/core/trunk/src/contrib/vaidya:776175-786373 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/examples:1152502-1230204 /hadoop/core/branches/branch-0.19/mapred/src/examples:713112 /hadoop/core/trunk/src/examples:776175-784663 Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 11 20:53:50 2012 @@ -1,3 +1,3 @@ -/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1229278 +/hadoop/common/trunk/hadoop-mapreduce-project/src/java:1152502-1230204 /hadoop/core/branches/branch-0.19/mapred/src/java:713112 /hadoop/core/trunk/src/mapred:776175-785643 Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/mapred-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/mapred-default.xml?rev=1230248&r1=1230247&r2=1230248&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/mapred-default.xml (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/mapred-default.xml Wed Jan 11 20:53:50 2012 @@ -960,6 +960,12 @@ + Network ACL to AM closed. + mapreduce.job.am-access-disabled + false + + + mapreduce.job.acl-modify-job Job specific access-control list for 'modifying' the job. It