hadoop-yarn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1619012 [4/26] - in /hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./ hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-api/ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/...
Date Tue, 19 Aug 2014 23:51:01 GMT
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
+import com.google.common.base.Supplier;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -27,19 +28,23 @@ import static org.mockito.Mockito.when;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 
 import org.junit.Assert;
-
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -71,9 +76,12 @@ import org.apache.hadoop.yarn.client.api
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.junit.After;
@@ -93,6 +101,9 @@ public class TestAMRMClient {
   static ApplicationAttemptId attemptId = null;
   static int nodeCount = 3;
   
+  static final int rolling_interval_sec = 13;
+  static final long am_expire_ms = 4000;
+
   static Resource capability;
   static Priority priority;
   static Priority priority2;
@@ -106,6 +117,10 @@ public class TestAMRMClient {
   public static void setup() throws Exception {
     // start minicluster
     conf = new YarnConfiguration();
+    conf.setLong(
+      YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+      rolling_interval_sec);
+    conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
     conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
     conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
     yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
@@ -800,6 +815,40 @@ public class TestAMRMClient {
     assertEquals(0, amClient.ask.size());
     assertEquals(0, amClient.release.size());
   }
+
+  class CountDownSupplier implements Supplier<Boolean> {
+    int counter = 0;
+    @Override
+    public Boolean get() {
+      counter++;
+      if (counter >= 3) {
+        return true;
+      } else {
+        return false;
+      }
+    }
+  };
+
+  @Test
+  public void testWaitFor() throws InterruptedException {
+    AMRMClientImpl<ContainerRequest> amClient = null;
+    CountDownSupplier countDownChecker = new CountDownSupplier();
+
+    try {
+      // start am rm client
+      amClient =
+          (AMRMClientImpl<ContainerRequest>) AMRMClient
+              .<ContainerRequest> createAMRMClient();
+      amClient.init(new YarnConfiguration());
+      amClient.start();
+      amClient.waitFor(countDownChecker, 1000);
+      assertEquals(3, countDownChecker.counter);
+    } finally {
+      if (amClient != null) {
+        amClient.stop();
+      }
+    }
+  }
   
   private void sleep(int sleepTime) {
     try {
@@ -809,4 +858,123 @@ public class TestAMRMClient {
     }
   }
 
+  @Test(timeout = 60000)
+  public void testAMRMClientOnAMRMTokenRollOver() throws YarnException,
+      IOException {
+    AMRMClient<ContainerRequest> amClient = null;
+    try {
+      AMRMTokenSecretManager amrmTokenSecretManager =
+          yarnCluster.getResourceManager().getRMContext()
+            .getAMRMTokenSecretManager();
+
+      // start am rm client
+      amClient = AMRMClient.<ContainerRequest> createAMRMClient();
+
+      amClient.init(conf);
+      amClient.start();
+
+      Long startTime = System.currentTimeMillis();
+      amClient.registerApplicationMaster("Host", 10000, "");
+
+      org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 =
+          getAMRMToken();
+      Assert.assertNotNull(amrmToken_1);
+      Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(),
+        amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+
+      // Wait for enough time and make sure the roll_over happens
+      // At mean time, the old AMRMToken should continue to work
+      while (System.currentTimeMillis() - startTime <
+          rolling_interval_sec * 1000) {
+        amClient.allocate(0.1f);
+        try {
+          Thread.sleep(1000);
+        } catch (InterruptedException e) {
+          // TODO Auto-generated catch block
+          e.printStackTrace();
+        }
+      }
+      amClient.allocate(0.1f);
+
+      org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 =
+          getAMRMToken();
+      Assert.assertNotNull(amrmToken_2);
+      Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(),
+        amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+
+      Assert.assertNotEquals(amrmToken_1, amrmToken_2);
+
+      // can do the allocate call with latest AMRMToken
+      amClient.allocate(0.1f);
+
+      // Make sure previous token has been rolled-over
+      // and can not use this rolled-over token to make a allocate all.
+      while (true) {
+        if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager
+          .getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
+          if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
+            break;
+          } else if (amrmToken_2.decodeIdentifier().getKeyId() !=
+              amrmTokenSecretManager.getNextMasterKeyData().getMasterKey()
+              .getKeyId()) {
+            break;
+          }
+        }
+        amClient.allocate(0.1f);
+        try {
+          Thread.sleep(1000);
+        } catch (InterruptedException e) {
+          // DO NOTHING
+        }
+      }
+
+      try {
+        UserGroupInformation testUser =
+            UserGroupInformation.createRemoteUser("testUser");
+        SecurityUtil.setTokenService(amrmToken_2, yarnCluster
+          .getResourceManager().getApplicationMasterService().getBindAddress());
+        testUser.addToken(amrmToken_2);
+        testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
+          @Override
+          public ApplicationMasterProtocol run() {
+            return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(
+              ApplicationMasterProtocol.class,
+              yarnCluster.getResourceManager().getApplicationMasterService()
+                .getBindAddress(), conf);
+          }
+        }).allocate(Records.newRecord(AllocateRequest.class));
+        Assert.fail("The old Token should not work");
+      } catch (Exception ex) {
+        Assert.assertTrue(ex instanceof InvalidToken);
+        Assert.assertTrue(ex.getMessage().contains(
+          "Invalid AMRMToken from "
+              + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
+      }
+
+      amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
+        null, null);
+
+    } finally {
+      if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+        amClient.stop();
+      }
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>
+      getAMRMToken() throws IOException {
+    Credentials credentials =
+        UserGroupInformation.getCurrentUser().getCredentials();
+    Iterator<org.apache.hadoop.security.token.Token<?>> iter =
+        credentials.getAllTokens().iterator();
+    while (iter.hasNext()) {
+      org.apache.hadoop.security.token.Token<?> token = iter.next();
+      if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
+        return (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>)
+            token;
+      }
+    }
+    return null;
+  }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Tue Aug 19 23:49:39 2014
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -346,7 +347,7 @@ public class TestNMClient {
           // 137 is possible if the container is not terminated but killed
           testGetContainerStatus(container, i, ContainerState.COMPLETE,
               "Container killed by the ApplicationMaster.", Arrays.asList(
-                  new Integer[] {137, 143, 0}));
+                  new Integer[] {ContainerExitStatus.KILLED_BY_APPMASTER}));
         } catch (YarnException e) {
           // The exception is possible because, after the container is stopped,
           // it may be removed from NM's context.

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Tue Aug 19 23:49:39 2014
@@ -25,19 +25,26 @@ import static org.mockito.Mockito.verify
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.junit.Assert;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@@ -69,19 +76,23 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestYarnClient {
@@ -725,4 +736,80 @@ public class TestYarnClient {
       IOUtils.closeQuietly(client);
     }
   }
+
+  @Test
+  public void testAutomaticTimelineDelegationTokenLoading()
+      throws Exception {
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+    final Token<TimelineDelegationTokenIdentifier> dToken =
+        new Token<TimelineDelegationTokenIdentifier>();
+    // crate a mock client
+    YarnClientImpl client = new YarnClientImpl() {
+      @Override
+      protected void serviceInit(Configuration conf) throws Exception {
+        if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+          timelineServiceEnabled = true;
+          timelineClient = mock(TimelineClient.class);
+          when(timelineClient.getDelegationToken(any(String.class)))
+              .thenReturn(dToken);
+          timelineClient.init(getConfig());
+          timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
+        }
+        this.setConfig(conf);
+      }
+
+      @Override
+      protected void serviceStart() throws Exception {
+        rmClient = mock(ApplicationClientProtocol.class);
+      }
+
+      @Override
+      protected void serviceStop() throws Exception {
+      }
+
+      @Override
+      public ApplicationReport getApplicationReport(ApplicationId appId) {
+        ApplicationReport report = mock(ApplicationReport.class);
+        when(report.getYarnApplicationState())
+            .thenReturn(YarnApplicationState.SUBMITTED);
+        return report;
+      }
+
+      @Override
+      public boolean isSecurityEnabled() {
+        return true;
+      }
+    };
+    client.init(conf);
+    client.start();
+    ApplicationSubmissionContext context =
+        mock(ApplicationSubmissionContext.class);
+    ApplicationId applicationId = ApplicationId.newInstance(0, 1);
+    when(context.getApplicationId()).thenReturn(applicationId);
+    DataOutputBuffer dob = new DataOutputBuffer();
+    Credentials credentials = new Credentials();
+    credentials.writeTokenStorageToStream(dob);
+    ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+    ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+        null, null, null, null, tokens, null);
+    when(context.getAMContainerSpec()).thenReturn(clc);
+    client.submitApplication(context);
+    // Check whether token is added or not
+    credentials = new Credentials();
+    DataInputByteBuffer dibb = new DataInputByteBuffer();
+    tokens = clc.getTokens();
+    if (tokens != null) {
+      dibb.reset(tokens);
+      credentials.readTokenStorageStream(dibb);
+      tokens.rewind();
+    }
+    Collection<Token<? extends TokenIdentifier>> dTokens =
+        credentials.getAllTokens();
+    Assert.assertEquals(1, dTokens.size());
+    Assert.assertEquals(dToken, dTokens.iterator().next());
+    client.stop();
+  }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Tue Aug 19 23:49:39 2014
@@ -38,24 +38,6 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <scope>provided</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>commons-el</groupId>
-          <artifactId>commons-el</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>tomcat</groupId>
-          <artifactId>jasper-runtime</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>tomcat</groupId>
-          <artifactId>jasper-compiler</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.mortbay.jetty</groupId>
-          <artifactId>jsp-2.1-jetty</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
 
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
@@ -85,12 +67,32 @@
       <artifactId>commons-codec</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+    </dependency>
+    <dependency>
       <groupId>com.sun.jersey</groupId>
       <artifactId>jersey-core</artifactId>
     </dependency>
     <dependency>
+      <groupId>com.sun.jersey</groupId>
+      <artifactId>jersey-client</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.codehaus.jackson</groupId>
-      <artifactId>jackson-mapper-asl</artifactId>
+      <artifactId>jackson-core-asl</artifactId>
+    </dependency>
+    <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-mapper-asl</artifactId>
+    </dependency>
+    <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-jaxrs</artifactId>
+    </dependency>
+    <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-xc</artifactId>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java Tue Aug 19 23:49:39 2014
@@ -25,6 +25,7 @@ import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.AMCommand;
 import org.apache.hadoop.yarn.api.records.Container;
@@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.PreemptionMessage;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl;
@@ -44,6 +46,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
 import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
 import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto;
@@ -74,7 +77,7 @@ public class AllocateResponsePBImpl exte
 
   private List<NodeReport> updatedNodes = null;
   private PreemptionMessage preempt;
-  
+  private Token amrmToken = null;
   
   public AllocateResponsePBImpl() {
     builder = AllocateResponseProto.newBuilder();
@@ -154,6 +157,9 @@ public class AllocateResponsePBImpl exte
           getChangeProtoIterable(this.decreasedContainers);
       builder.addAllDecreasedContainers(iterable);
     }
+    if (this.amrmToken != null) {
+      builder.setAmRmToken(convertToProtoFormat(this.amrmToken));
+    }
   }
 
   private synchronized void mergeLocalToProto() {
@@ -357,6 +363,28 @@ public class AllocateResponsePBImpl exte
     this.decreasedContainers.addAll(decreasedContainers);
   }
 
+  @Override
+  public synchronized Token getAMRMToken() {
+    AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+    if (amrmToken != null) {
+      return amrmToken;
+    }
+    if (!p.hasAmRmToken()) {
+      return null;
+    }
+    this.amrmToken = convertFromProtoFormat(p.getAmRmToken());
+    return amrmToken;
+  }
+
+  @Override
+  public synchronized void setAMRMToken(Token amRMToken) {
+    maybeInitBuilder();
+    if (amRMToken == null) {
+      builder.clearAmRmToken();
+    }
+    this.amrmToken = amRMToken;
+  }
+
   private synchronized void initLocalIncreasedContainerList() {
     if (this.increasedContainers != null) {
       return;
@@ -699,4 +727,12 @@ public class AllocateResponsePBImpl exte
   private synchronized NMToken convertFromProtoFormat(NMTokenProto proto) {
     return new NMTokenPBImpl(proto);
   }
+
+  private TokenPBImpl convertFromProtoFormat(TokenProto p) {
+    return new TokenPBImpl(p);
+  }
+
+  private TokenProto convertToProtoFormat(Token t) {
+    return ((TokenPBImpl)t).getProto();
+  }
 }  

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Tue Aug 19 23:49:39 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.api.proto
 
 import java.util.EnumSet;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
@@ -35,6 +34,8 @@ import org.apache.hadoop.yarn.proto.Yarn
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
 import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
 
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
 import com.google.protobuf.TextFormat;
 
 @Private
@@ -49,7 +50,8 @@ public class GetApplicationsRequestPBImp
   Set<String> users = null;
   Set<String> queues = null;
   long limit = Long.MAX_VALUE;
-  LongRange start = null, finish = null;
+  LongRange start = null;
+  LongRange finish = null;
   private Set<String> applicationTags;
   private ApplicationsRequestScope scope;
 
@@ -78,57 +80,46 @@ public class GetApplicationsRequestPBImp
   }
 
   private void mergeLocalToBuilder() {
-    if (this.applicationTypes != null) {
-      addLocalApplicationTypesToProto();
+    if (applicationTypes != null && !applicationTypes.isEmpty()) {
+      builder.clearApplicationTypes();
+      builder.addAllApplicationTypes(applicationTypes);
     }
-    if (this.applicationStates != null) {
-      maybeInitBuilder();
+    if (applicationStates != null && !applicationStates.isEmpty()) {
       builder.clearApplicationStates();
-      Iterable<YarnApplicationStateProto> iterable =
-          new Iterable<YarnApplicationStateProto>() {
-
+      builder.addAllApplicationStates(Iterables.transform(applicationStates,
+          new Function<YarnApplicationState, YarnApplicationStateProto>() {
             @Override
-            public Iterator<YarnApplicationStateProto> iterator() {
-              return new Iterator<YarnApplicationStateProto>() {
-
-                Iterator<YarnApplicationState> iter = applicationStates
-                    .iterator();
-
-                @Override
-                public boolean hasNext() {
-                  return iter.hasNext();
-                }
-
-                @Override
-                public YarnApplicationStateProto next() {
-                  return ProtoUtils.convertToProtoFormat(iter.next());
-                }
-
-                @Override
-                public void remove() {
-                  throw new UnsupportedOperationException();
-
-                }
-              };
-
+            public YarnApplicationStateProto apply(YarnApplicationState input) {
+              return ProtoUtils.convertToProtoFormat(input);
             }
-          };
-      builder.addAllApplicationStates(iterable);
+          }));
     }
-    if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+    if (applicationTags != null && !applicationTags.isEmpty()) {
+      builder.clearApplicationTags();
       builder.addAllApplicationTags(this.applicationTags);
     }
-    if (this.scope != null) {
+    if (scope != null) {
       builder.setScope(ProtoUtils.convertToProtoFormat(scope));
     }
-  }
-
-  private void addLocalApplicationTypesToProto() {
-    maybeInitBuilder();
-    builder.clearApplicationTypes();
-    if (this.applicationTypes == null)
-      return;
-    builder.addAllApplicationTypes(applicationTypes);
+    if (start != null) {
+      builder.setStartBegin(start.getMinimumLong());
+      builder.setStartEnd(start.getMaximumLong());
+    }
+    if (finish != null) {
+      builder.setFinishBegin(finish.getMinimumLong());
+      builder.setFinishEnd(finish.getMaximumLong());
+    }
+    if (limit != Long.MAX_VALUE) {
+      builder.setLimit(limit);
+    }
+    if (users != null && !users.isEmpty()) {
+      builder.clearUsers();
+      builder.addAllUsers(users);
+    }
+    if (queues != null && !queues.isEmpty()) {
+      builder.clearQueues();
+      builder.addAllQueues(queues);
+    }
   }
 
   private void maybeInitBuilder() {
@@ -326,7 +317,7 @@ public class GetApplicationsRequestPBImp
   public LongRange getStartRange() {
     if (this.start == null) {
       GetApplicationsRequestProtoOrBuilder p = viaProto ? proto: builder;
-      if (p.hasStartBegin() || p.hasFinishBegin()) {
+      if (p.hasStartBegin() || p.hasStartEnd()) {
         long begin = p.hasStartBegin() ? p.getStartBegin() : 0L;
         long end = p.hasStartEnd() ? p.getStartEnd() : Long.MAX_VALUE;
         this.start = new LongRange(begin, end);
@@ -336,6 +327,11 @@ public class GetApplicationsRequestPBImp
   }
 
   @Override
+  public void setStartRange(LongRange range) {
+    this.start = range;
+  }
+
+  @Override
   public void setStartRange(long begin, long end)
       throws IllegalArgumentException {
     if (begin > end) {
@@ -359,6 +355,11 @@ public class GetApplicationsRequestPBImp
   }
 
   @Override
+  public void setFinishRange(LongRange range) {
+    this.finish = range;
+  }
+
+  @Override
   public void setFinishRange(long begin, long end) {
     if (begin > end) {
       throw new IllegalArgumentException("begin > end in range (begin, " +

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Tue Aug 19 23:49:39 2014
@@ -479,6 +479,7 @@ public class ApplicationReportPBImpl ext
       builder.setAmRmToken(convertToProtoFormat(this.amRmToken));
     }
     if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+      builder.clearApplicationTags();
       builder.addAllApplicationTags(this.applicationTags);
     }
   }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java Tue Aug 19 23:49:39 2014
@@ -107,6 +107,7 @@ extends ApplicationSubmissionContext {
       builder.setResource(convertToProtoFormat(this.resource));
     }
     if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+      builder.clearApplicationTags();
       builder.addAllApplicationTags(this.applicationTags);
     }
   }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java Tue Aug 19 23:49:39 2014
@@ -18,24 +18,19 @@
 
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
-
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerReportProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ContainerReportProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
 
 import com.google.protobuf.TextFormat;
 
@@ -150,9 +145,9 @@ public class ContainerReportPBImpl exten
   }
 
   @Override
-  public long getStartTime() {
+  public long getCreationTime() {
     ContainerReportProtoOrBuilder p = viaProto ? proto : builder;
-    return p.getStartTime();
+    return p.getCreationTime();
   }
 
   @Override
@@ -237,9 +232,9 @@ public class ContainerReportPBImpl exten
   }
 
   @Override
-  public void setStartTime(long startTime) {
+  public void setCreationTime(long creationTime) {
     maybeInitBuilder();
-    builder.setStartTime(startTime);
+    builder.setCreationTime(creationTime);
   }
 
   public ContainerReportProto getProto() {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java Tue Aug 19 23:49:39 2014
@@ -90,7 +90,7 @@ public class ResourceBlacklistRequestPBI
 
   private void addBlacklistRemovalsToProto() {
     maybeInitBuilder();
-    builder.clearBlacklistAdditions();
+    builder.clearBlacklistRemovals();
     if (this.blacklistRemovals == null) { 
       return;
     }
@@ -159,5 +159,14 @@ public class ResourceBlacklistRequestPBI
   public int hashCode() {
     return getProto().hashCode();
   }
-  
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null)
+      return false;
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java Tue Aug 19 23:49:39 2014
@@ -86,4 +86,19 @@ public class ResourceOptionPBImpl extend
     builder = null;
   }
 
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null)
+      return false;
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java Tue Aug 19 23:49:39 2014
@@ -32,9 +32,9 @@ import org.apache.hadoop.yarn.proto.Yarn
 
 public class SerializedExceptionPBImpl extends SerializedException {
 
-  SerializedExceptionProto proto = SerializedExceptionProto
-      .getDefaultInstance();
-  SerializedExceptionProto.Builder builder = null;
+  SerializedExceptionProto proto = null;
+  SerializedExceptionProto.Builder builder =
+      SerializedExceptionProto.newBuilder();
   boolean viaProto = false;
 
   public SerializedExceptionPBImpl() {
@@ -135,6 +135,22 @@ public class SerializedExceptionPBImpl e
     return proto;
   }
 
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null) {
+      return false;
+    }
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
+
   private void maybeInitBuilder() {
     if (viaProto || builder == null) {
       builder = SerializedExceptionProto.newBuilder(proto);

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java Tue Aug 19 23:49:39 2014
@@ -48,7 +48,7 @@ public class TokenPBImpl extends Token {
   }
 
   public synchronized TokenProto getProto() {
-      mergeLocalToProto();
+    mergeLocalToProto();
     proto = viaProto ? proto : builder.build();
     viaProto = true;
     return proto;

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java Tue Aug 19 23:49:39 2014
@@ -109,7 +109,10 @@ public class LogCLIHelpers implements Co
         remoteRootLogDir, appId, user, logDirSuffix);
     RemoteIterator<FileStatus> nodeFiles;
     try {
-      nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir);
+      Path qualifiedLogDir =
+          FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir);
+      nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(),
+          getConf()).listStatus(remoteAppLogDir);
     } catch (FileNotFoundException fnf) {
       System.out.println("Logs not available at " + remoteAppLogDir.toString());
       System.out

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java Tue Aug 19 23:49:39 2014
@@ -44,6 +44,7 @@ public class AMRMTokenIdentifier extends
   public static final Text KIND_NAME = new Text("YARN_AM_RM_TOKEN");
 
   private ApplicationAttemptId applicationAttemptId;
+  private int keyId = Integer.MIN_VALUE;
 
   public AMRMTokenIdentifier() {
   }
@@ -53,6 +54,13 @@ public class AMRMTokenIdentifier extends
     this.applicationAttemptId = appAttemptId;
   }
 
+  public AMRMTokenIdentifier(ApplicationAttemptId appAttemptId,
+      int masterKeyId) {
+    this();
+    this.applicationAttemptId = appAttemptId;
+    this.keyId = masterKeyId;
+  }
+
   @Private
   public ApplicationAttemptId getApplicationAttemptId() {
     return this.applicationAttemptId;
@@ -64,6 +72,7 @@ public class AMRMTokenIdentifier extends
     out.writeLong(appId.getClusterTimestamp());
     out.writeInt(appId.getId());
     out.writeInt(this.applicationAttemptId.getAttemptId());
+    out.writeInt(this.keyId);
   }
 
   @Override
@@ -75,6 +84,7 @@ public class AMRMTokenIdentifier extends
         ApplicationId.newInstance(clusterTimeStamp, appId);
     this.applicationAttemptId =
         ApplicationAttemptId.newInstance(applicationId, attemptId);
+    this.keyId = in.readInt();
   }
 
   @Override
@@ -92,6 +102,10 @@ public class AMRMTokenIdentifier extends
         .toString());
   }
 
+  public int getKeyId() {
+    return this.keyId;
+  }
+
   // TODO: Needed?
   @InterfaceAudience.Private
   public static class Renewer extends Token.TrivialRenewer {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Tue Aug 19 23:49:39 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 
 /**
@@ -56,10 +57,12 @@ public class ContainerTokenIdentifier ex
   private long expiryTimeStamp;
   private int masterKeyId;
   private long rmIdentifier;
+  private Priority priority;
+  private long creationTime;
 
-  public ContainerTokenIdentifier(ContainerId containerID, String hostName,
-      String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId,
-      long rmIdentifier) {
+  public ContainerTokenIdentifier(ContainerId containerID,
+      String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
+      int masterKeyId, long rmIdentifier, Priority priority, long creationTime) {
     this.containerId = containerID;
     this.nmHostAddr = hostName;
     this.appSubmitter = appSubmitter;
@@ -67,6 +70,8 @@ public class ContainerTokenIdentifier ex
     this.expiryTimeStamp = expiryTimeStamp;
     this.masterKeyId = masterKeyId;
     this.rmIdentifier = rmIdentifier;
+    this.priority = priority;
+    this.creationTime = creationTime;
   }
 
   /**
@@ -99,6 +104,13 @@ public class ContainerTokenIdentifier ex
     return this.masterKeyId;
   }
 
+  public Priority getPriority() {
+    return this.priority;
+  }
+
+  public long getCreationTime() {
+    return this.creationTime;
+  }
   /**
    * Get the RMIdentifier of RM in which containers are allocated
    * @return RMIdentifier
@@ -124,6 +136,8 @@ public class ContainerTokenIdentifier ex
     out.writeLong(this.expiryTimeStamp);
     out.writeInt(this.masterKeyId);
     out.writeLong(this.rmIdentifier);
+    out.writeInt(this.priority.getPriority());
+    out.writeLong(this.creationTime);
   }
 
   @Override
@@ -142,6 +156,8 @@ public class ContainerTokenIdentifier ex
     this.expiryTimeStamp = in.readLong();
     this.masterKeyId = in.readInt();
     this.rmIdentifier = in.readLong();
+    this.priority = Priority.newInstance(in.readInt());
+    this.creationTime = in.readLong();
   }
 
   @Override

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java Tue Aug 19 23:49:39 2014
@@ -162,5 +162,19 @@ public class UpdateNodeResourceRequestPB
     };
     this.builder.addAllNodeResourceMap(values);
   }
-  
+
+  @Override
+  public int hashCode() {
+    return getProto().hashCode();
+  }
+
+  @Override
+  public boolean equals(Object other) {
+    if (other == null)
+      return false;
+    if (other.getClass().isAssignableFrom(this.getClass())) {
+      return this.getProto().equals(this.getClass().cast(other).getProto());
+    }
+    return false;
+  }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java Tue Aug 19 23:49:39 2014
@@ -173,7 +173,7 @@ public class ApplicationClassLoader exte
   }
 
   @VisibleForTesting
-  static boolean isSystemClass(String name, List<String> systemClasses) {
+  public static boolean isSystemClass(String name, List<String> systemClasses) {
     if (systemClasses != null) {
       String canonicalName = name.replace('/', '.');
       while (canonicalName.startsWith(".")) {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Tue Aug 19 23:49:39 2014
@@ -177,9 +177,10 @@ public class FSDownload implements Calla
   /**
    * Returns true if all ancestors of the specified path have the 'execute'
    * permission set for all users (i.e. that other users can traverse
-   * the directory heirarchy to the given path)
+   * the directory hierarchy to the given path)
    */
-  private static boolean ancestorsHaveExecutePermissions(FileSystem fs,
+  @VisibleForTesting
+  static boolean ancestorsHaveExecutePermissions(FileSystem fs,
       Path path, LoadingCache<Path,Future<FileStatus>> statCache)
       throws IOException {
     Path current = path;

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java Tue Aug 19 23:49:39 2014
@@ -21,10 +21,14 @@ package org.apache.hadoop.yarn.util;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 
 @Private
 public class Times {
+  private static final Log LOG = LogFactory.getLog(Times.class);
+
   static final ThreadLocal<SimpleDateFormat> dateFormat =
       new ThreadLocal<SimpleDateFormat>() {
         @Override protected SimpleDateFormat initialValue() {
@@ -36,12 +40,30 @@ public class Times {
     return Times.elapsed(started, finished, true);
   }
 
+  // A valid elapsed is supposed to be non-negative. If finished/current time
+  // is ahead of the started time, return -1 to indicate invalid elapsed time,
+  // and record a warning log.
   public static long elapsed(long started, long finished, boolean isRunning) {
     if (finished > 0 && started > 0) {
-      return finished - started;
+      long elapsed = finished - started;
+      if (elapsed >= 0) {
+        return elapsed;
+      } else {
+        LOG.warn("Finished time " + finished
+            + " is ahead of started time " + started);
+        return -1;
+      }
     }
     if (isRunning) {
-      return started > 0 ? System.currentTimeMillis() - started : 0;
+      long current = System.currentTimeMillis();
+      long elapsed = started > 0 ? current - started : 0;
+      if (elapsed >= 0) {
+        return elapsed;
+      } else {
+        LOG.warn("Current time " + current
+            + " is ahead of started time " + started);
+        return -1;
+      }
     } else {
       return -1;
     }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java Tue Aug 19 23:49:39 2014
@@ -42,6 +42,13 @@ public class DefaultResourceCalculator e
       Resource numerator, Resource denominator) {
     return ratio(numerator, denominator);
   }
+  
+  public boolean isInvalidDivisor(Resource r) {
+    if (r.getMemory() == 0.0f) {
+      return true;
+    }
+    return false;
+  }
 
   @Override
   public float ratio(Resource a, Resource b) {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java Tue Aug 19 23:49:39 2014
@@ -109,6 +109,14 @@ public class DominantResourceCalculator 
         getResourceAsValue(clusterResource, numerator, true) / 
         getResourceAsValue(clusterResource, denominator, true);
   }
+  
+  @Override
+  public boolean isInvalidDivisor(Resource r) {
+    if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) {
+      return true;
+    }
+    return false;
+  }
 
   @Override
   public float ratio(Resource a, Resource b) {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java Tue Aug 19 23:49:39 2014
@@ -150,6 +150,15 @@ public abstract class ResourceCalculator
       Resource clusterResource, Resource numerator, Resource denominator);
   
   /**
+   * Determine if a resource is not suitable for use as a divisor
+   * (will result in divide by 0, etc)
+   *
+   * @param r resource
+   * @return true if divisor is invalid (should not be used), false else
+   */
+  public abstract boolean isInvalidDivisor(Resource r);
+
+  /**
    * Ratio of resource <code>a</code> to resource <code>b</code>.
    * 
    * @param a resource 

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java Tue Aug 19 23:49:39 2014
@@ -184,6 +184,11 @@ public class Resources {
     return calculator.roundDown(lhs, factor);
   }
   
+  public static boolean isInvalidDivisor(
+      ResourceCalculator resourceCalculator, Resource divisor) {
+    return resourceCalculator.isInvalidDivisor(divisor);
+  }
+
   public static float ratio(
       ResourceCalculator resourceCalculator, Resource lhs, Resource rhs) {
     return resourceCalculator.ratio(lhs, rhs);

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java Tue Aug 19 23:49:39 2014
@@ -19,15 +19,18 @@
 package org.apache.hadoop.yarn.util.timeline;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
 import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.map.AnnotationIntrospector;
 import org.codehaus.jackson.map.JsonMappingException;
 import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion;
-import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
 
 /**
  * The helper class for the timeline module.
@@ -41,9 +44,7 @@ public class TimelineUtils {
 
   static {
     mapper = new ObjectMapper();
-    AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
-    mapper.setAnnotationIntrospector(introspector);
-    mapper.setSerializationInclusion(Inclusion.NON_NULL);
+    YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
   }
 
   /**
@@ -82,4 +83,26 @@ public class TimelineUtils {
     }
   }
 
+  public static InetSocketAddress getTimelineTokenServiceAddress(
+      Configuration conf) {
+    InetSocketAddress timelineServiceAddr = null;
+    if (YarnConfiguration.useHttps(conf)) {
+      timelineServiceAddr = conf.getSocketAddr(
+          YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT);
+    } else {
+      timelineServiceAddr = conf.getSocketAddr(
+          YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT);
+    }
+    return timelineServiceAddr;
+  }
+
+  public static Text buildTimelineTokenService(Configuration conf) {
+    InetSocketAddress timelineServiceAddr =
+        getTimelineTokenServiceAddress(conf);
+    return SecurityUtil.buildTokenService(timelineServiceAddr);
+  }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java Tue Aug 19 23:49:39 2014
@@ -21,10 +21,12 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 
 import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.ext.ExceptionMapper;
 import javax.ws.rs.ext.Provider;
+import javax.xml.bind.UnmarshalException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -79,6 +81,8 @@ public class GenericExceptionHandler imp
       s = Response.Status.NOT_FOUND;
     } else if (e instanceof IOException) {
       s = Response.Status.NOT_FOUND;
+    } else if (e instanceof ForbiddenException) {
+      s = Response.Status.FORBIDDEN;
     } else if (e instanceof UnsupportedOperationException) {
       s = Response.Status.BAD_REQUEST;
     } else if (e instanceof IllegalArgumentException) {
@@ -87,6 +91,9 @@ public class GenericExceptionHandler imp
       s = Response.Status.BAD_REQUEST;
     } else if (e instanceof BadRequestException) {
       s = Response.Status.BAD_REQUEST;
+    } else if (e instanceof WebApplicationException
+        && e.getCause() instanceof UnmarshalException) {
+      s = Response.Status.BAD_REQUEST;
     } else {
       LOG.warn("INTERNAL_SERVER_ERROR", e);
       s = Response.Status.INTERNAL_SERVER_ERROR;

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java Tue Aug 19 23:49:39 2014
@@ -49,9 +49,14 @@ public class YarnJacksonJaxbJsonProvider
   @Override
   public ObjectMapper locateMapper(Class<?> type, MediaType mediaType) {
     ObjectMapper mapper = super.locateMapper(type, mediaType);
+    configObjectMapper(mapper);
+    return mapper;
+  }
+
+  public static void configObjectMapper(ObjectMapper mapper) {
     AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
     mapper.setAnnotationIntrospector(introspector);
     mapper.setSerializationInclusion(Inclusion.NON_NULL);
-    return mapper;
   }
+
 }
\ No newline at end of file

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java Tue Aug 19 23:49:39 2014
@@ -19,12 +19,12 @@ package org.apache.hadoop.yarn.webapp.ut
 
 import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -34,11 +34,18 @@ import org.apache.hadoop.http.HttpServer
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.util.RMHAUtils;
 
 @Private
 @Evolving
 public class WebAppUtils {
+  public static final String WEB_APP_TRUSTSTORE_PASSWORD_KEY =
+      "ssl.server.truststore.password";
+  public static final String WEB_APP_KEYSTORE_PASSWORD_KEY =
+      "ssl.server.keystore.password";
+  public static final String WEB_APP_KEY_PASSWORD_KEY =
+      "ssl.server.keystore.keypassword";
   public static final String HTTPS_PREFIX = "https://";
   public static final String HTTP_PREFIX = "http://";
 
@@ -170,6 +177,37 @@ public class WebAppUtils {
     return sb.toString();
   }
   
+  /**
+   * Get the URL to use for binding where bind hostname can be specified
+   * to override the hostname in the webAppURLWithoutScheme. Port specified in the
+   * webAppURLWithoutScheme will be used.
+   *
+   * @param conf the configuration
+   * @param hostProperty bind host property name
+   * @param webAppURLWithoutScheme web app URL without scheme String
+   * @return String representing bind URL
+   */
+  public static String getWebAppBindURL(
+      Configuration conf,
+      String hostProperty,
+      String webAppURLWithoutScheme) {
+
+    // If the bind-host setting exists then it overrides the hostname
+    // portion of the corresponding webAppURLWithoutScheme
+    String host = conf.getTrimmed(hostProperty);
+    if (host != null && !host.isEmpty()) {
+      if (webAppURLWithoutScheme.contains(":")) {
+        webAppURLWithoutScheme = host + ":" + webAppURLWithoutScheme.split(":")[1];
+      }
+      else {
+        throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " +
+                                       webAppURLWithoutScheme);
+      }
+    }
+
+    return webAppURLWithoutScheme;
+  }
+
   public static String getNMWebAppURLWithoutScheme(Configuration conf) {
     if (YarnConfiguration.useHttps(conf)) {
       return conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
@@ -242,21 +280,56 @@ public class WebAppUtils {
 
   /**
    * Load the SSL keystore / truststore into the HttpServer builder.
+   * @param builder the HttpServer2.Builder to populate with ssl config
    */
   public static HttpServer2.Builder loadSslConfiguration(
       HttpServer2.Builder builder) {
-    Configuration sslConf = new Configuration(false);
+    return loadSslConfiguration(builder, null);
+  }
+
+  /**
+   * Load the SSL keystore / truststore into the HttpServer builder.
+   * @param builder the HttpServer2.Builder to populate with ssl config
+   * @param sslConf the Configuration instance to use during loading of SSL conf
+   */
+  public static HttpServer2.Builder loadSslConfiguration(
+      HttpServer2.Builder builder, Configuration sslConf) {
+    if (sslConf == null) {
+      sslConf = new Configuration(false);
+    }
     boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
     sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT);
 
     return builder
         .needsClientAuth(needsClientAuth)
-        .keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
+        .keyPassword(getPassword(sslConf, WEB_APP_KEY_PASSWORD_KEY))
         .keyStore(sslConf.get("ssl.server.keystore.location"),
-            sslConf.get("ssl.server.keystore.password"),
+            getPassword(sslConf, WEB_APP_KEYSTORE_PASSWORD_KEY),
             sslConf.get("ssl.server.keystore.type", "jks"))
         .trustStore(sslConf.get("ssl.server.truststore.location"),
-            sslConf.get("ssl.server.truststore.password"),
+            getPassword(sslConf, WEB_APP_TRUSTSTORE_PASSWORD_KEY),
             sslConf.get("ssl.server.truststore.type", "jks"));
   }
+
+  /**
+   * Leverages the Configuration.getPassword method to attempt to get
+   * passwords from the CredentialProvider API before falling back to
+   * clear text in config - if falling back is allowed.
+   * @param conf Configuration instance
+   * @param alias name of the credential to retreive
+   * @return String credential value or null
+   */
+  static String getPassword(Configuration conf, String alias) {
+    String password = null;
+    try {
+      char[] passchars = conf.getPassword(alias);
+      if (passchars != null) {
+        password = new String(passchars);
+      }
+    }
+    catch (IOException ioe) {
+      password = null;
+    }
+    return password;
+  }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java Tue Aug 19 23:49:39 2014
@@ -25,7 +25,6 @@ public class FooterBlock extends HtmlBlo
 
   @Override protected void render(Block html) {
     html.
-      div("#footer.ui-widget").
-        a("http://hadoop.apache.org/", "About Apache Hadoop")._();
+      div("#footer.ui-widget")._();
   }
 }

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java Tue Aug 19 23:49:39 2014
@@ -79,7 +79,8 @@ public abstract class HtmlPage extends T
   @Override
   public void render() {
     puts(DOCTYPE);
-    render(page().html().meta_http("Content-type", MimeType.HTML));
+    render(page().html().meta_http("X-UA-Compatible", "IE=8")
+        .meta_http("Content-type", MimeType.HTML));
     if (page().nestLevel() != 0) {
       throw new WebAppException("Error rendering page: nestLevel="+
                                 page().nestLevel());

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java Tue Aug 19 23:49:39 2014
@@ -62,11 +62,11 @@ public class InfoBlock extends HtmlBlock
         	DIV<TD<TR<TABLE<DIV<Hamlet>>>>> singleLineDiv;
             for ( String line :lines) {
               singleLineDiv = td.div();
-              singleLineDiv._r(line);
+              singleLineDiv._(line);
               singleLineDiv._();
             }
           } else {
-            td._r(value);
+            td._(value);
           }
           td._();
         } else {

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo Tue Aug 19 23:49:39 2014
@@ -12,6 +12,7 @@
 #   limitations under the License.
 #
 org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo
+org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo
 org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo
 org.apache.hadoop.yarn.security.SchedulerSecurityInfo
 org.apache.hadoop.yarn.security.admin.AdminSecurityInfo

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier Tue Aug 19 23:49:39 2014
@@ -15,4 +15,5 @@ org.apache.hadoop.yarn.security.Containe
 org.apache.hadoop.yarn.security.AMRMTokenIdentifier
 org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier
 org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier
+org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier
 org.apache.hadoop.yarn.security.NMTokenIdentifier

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer Tue Aug 19 23:49:39 2014
@@ -14,3 +14,4 @@
 org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer
 org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer
 org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer
+org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier$Renewer

Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js Tue Aug 19 23:49:39 2014
@@ -119,13 +119,7 @@ function parseHadoopID(data, type, full)
     return data;
   }
   //Return the visible string rather than the entire HTML tag
-  if (type === 'filter') {
-    return data.split('>')[1].split('<')[0];
-  }
-  //Parse the ID for 'sort', 'type' and undefined
-  //The number after the last '_' and before the end tag '<'
-  var splits = data.split('_');
-  return splits[parseInt(splits.length-1)].split('<')[0];
+  return data.split('>')[1].split('<')[0];
 }
 
 //JSON array element is "20000 attempt_1360183373897_0001_m_000002_0"



Mime
View raw message