hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From junping...@apache.org
Subject [3/3] hadoop git commit: Revert "HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah."
Date Tue, 08 May 2018 00:33:54 GMT
Revert "HADOOP-14445. Delegation tokens are not shared between KMS instances. Contributed by Xiao Chen and Rushabh S Shah."

This reverts commit 714a079ffb88540ec1e09d5023c35e1fa0dd016d.

 Conflicts:
	hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java

(cherry picked from commit 9e4599fbd34b2f0a39f30a10c7e3eddc2bfd5a9e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d10c4a97
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d10c4a97
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d10c4a97

Branch: refs/heads/branch-2.8.4
Commit: d10c4a97e07751f4152dd36800d9d3bd6c50b531
Parents: 2ef1ace
Author: Xiao Chen <xiao@apache.org>
Authored: Mon May 7 11:21:26 2018 -0700
Committer: Junping Du <junping_du@apache.org>
Committed: Tue May 8 08:30:04 2018 +0800

----------------------------------------------------------------------
 .../crypto/key/kms/KMSClientProvider.java       | 216 ++++----
 .../crypto/key/kms/KMSDelegationToken.java      |  22 +-
 .../crypto/key/kms/KMSLegacyTokenRenewer.java   |  56 --
 .../hadoop/crypto/key/kms/KMSTokenRenewer.java  | 103 ----
 .../hadoop/crypto/key/kms/package-info.java     |  18 -
 .../fs/CommonConfigurationKeysPublic.java       |  10 -
 .../web/DelegationTokenAuthenticatedURL.java    |  21 +-
 .../DelegationTokenAuthenticationHandler.java   |   8 +-
 .../web/DelegationTokenAuthenticator.java       |   2 +-
 .../java/org/apache/hadoop/util/KMSUtil.java    |  45 +-
 .../hadoop/util/KMSUtilFaultInjector.java       |  49 --
 ...apache.hadoop.security.token.TokenIdentifier |   1 -
 ...rg.apache.hadoop.security.token.TokenRenewer |   3 +-
 .../src/main/resources/core-default.xml         |  20 -
 .../crypto/key/kms/TestKMSClientProvider.java   | 166 ------
 .../kms/TestLoadBalancingKMSClientProvider.java |  67 +--
 .../org/apache/hadoop/util/TestKMSUtil.java     |  65 ---
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 507 +++----------------
 18 files changed, 203 insertions(+), 1176 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index bd951f4..fc5a5da 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -36,11 +36,11 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.hadoop.security.token.TokenRenewer;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.hadoop.util.HttpExceptionUtils;
+import org.apache.hadoop.util.KMSUtil;
 import org.apache.http.client.utils.URIBuilder;
 import org.codehaus.jackson.map.ObjectMapper;
 import org.slf4j.Logger;
@@ -82,9 +82,6 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT;
-
 /**
  * KMS client <code>KeyProvider</code> implementation.
  */
@@ -92,13 +89,16 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.KMS_CLIENT_COPY
 public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     KeyProviderDelegationTokenExtension.DelegationTokenExtension {
 
-  public static final Logger LOG =
+  private static final Logger LOG =
       LoggerFactory.getLogger(KMSClientProvider.class);
 
   private static final String INVALID_SIGNATURE = "Invalid signature";
 
   private static final String ANONYMOUS_REQUESTS_DISALLOWED = "Anonymous requests are disallowed";
 
+  public static final String TOKEN_KIND_STR = KMSDelegationToken.TOKEN_KIND_STR;
+  public static final Text TOKEN_KIND = KMSDelegationToken.TOKEN_KIND;
+
   public static final String SCHEME_NAME = "kms";
 
   private static final String UTF8 = "UTF-8";
@@ -123,17 +123,12 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 
   private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue;
 
-  /* dtService defines the token service value for the kms token.
-   * The value can be legacy format which is ip:port format or it can be uri.
-   * If it's uri format, then the value is read from
-   * CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH at key
-   * provider creation time, and set to token's Service field.
-   * When a token is renewed / canceled, its Service field will be used to
-   * instantiate a KeyProvider, eliminating the need to read configs
-    * at that time.
-   */
   private final Text dtService;
-  private final boolean copyLegacyToken;
+
+  // Allow fallback to default kms server port 9600 for certain tests that do
+  // not specify the port explicitly in the kms provider url.
+  @VisibleForTesting
+  public static volatile boolean fallbackDefaultPortForTesting = false;
 
   private class EncryptedQueueRefiller implements
     ValueQueue.QueueRefiller<EncryptedKeyVersion> {
@@ -157,6 +152,68 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     }
   }
 
+  /**
+   * The KMS implementation of {@link TokenRenewer}.
+   */
+  public static class KMSTokenRenewer extends TokenRenewer {
+    private static final Logger LOG =
+        LoggerFactory.getLogger(KMSTokenRenewer.class);
+
+    @Override
+    public boolean handleKind(Text kind) {
+      return kind.equals(TOKEN_KIND);
+    }
+
+    @Override
+    public boolean isManaged(Token<?> token) throws IOException {
+      return true;
+    }
+
+    @Override
+    public long renew(Token<?> token, Configuration conf) throws IOException {
+      LOG.debug("Renewing delegation token {}", token);
+      KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
+          KeyProviderFactory.KEY_PROVIDER_PATH);
+      try {
+        if (!(keyProvider instanceof
+            KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+          throw new IOException(String
+              .format("keyProvider %s cannot renew token [%s]",
+                  keyProvider == null ? "null" : keyProvider.getClass(),
+                  token));
+        }
+        return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+            keyProvider).renewDelegationToken(token);
+      } finally {
+        if (keyProvider != null) {
+          keyProvider.close();
+        }
+      }
+    }
+
+    @Override
+    public void cancel(Token<?> token, Configuration conf) throws IOException {
+      LOG.debug("Canceling delegation token {}", token);
+      KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
+          KeyProviderFactory.KEY_PROVIDER_PATH);
+      try {
+        if (!(keyProvider instanceof
+            KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+          throw new IOException(String
+              .format("keyProvider %s cannot cancel token [%s]",
+                  keyProvider == null ? "null" : keyProvider.getClass(),
+                  token));
+        }
+        ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+            keyProvider).cancelDelegationToken(token);
+      } finally {
+        if (keyProvider != null) {
+          keyProvider.close();
+        }
+      }
+    }
+  }
+
   public static class KMSEncryptedKeyVersion extends EncryptedKeyVersion {
     public KMSEncryptedKeyVersion(String keyName, String keyVersionName,
         byte[] iv, String encryptedVersionName, byte[] keyMaterial) {
@@ -279,13 +336,13 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
           }
           hostsPart = t[0];
         }
-        return createProvider(conf, origUrl, port, hostsPart, providerUri);
+        return createProvider(conf, origUrl, port, hostsPart);
       }
       return null;
     }
 
-    private KeyProvider createProvider(Configuration conf, URL origUrl,
-        int port, String hostsPart, URI providerUri) throws IOException {
+    private KeyProvider createProvider(Configuration conf,
+        URL origUrl, int port, String hostsPart) throws IOException {
       String[] hosts = hostsPart.split(";");
       KMSClientProvider[] providers = new KMSClientProvider[hosts.length];
       for (int i = 0; i < hosts.length; i++) {
@@ -293,7 +350,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
           providers[i] =
               new KMSClientProvider(
                   new URI("kms", origUrl.getProtocol(), hosts[i], port,
-                      origUrl.getPath(), null, null), conf, providerUri);
+                      origUrl.getPath(), null, null), conf);
         } catch (URISyntaxException e) {
           throw new IOException("Could not instantiate KMSProvider.", e);
         }
@@ -370,10 +427,17 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     }
   }
 
-  public KMSClientProvider(URI uri, Configuration conf, URI providerUri) throws
-      IOException {
+  public KMSClientProvider(URI uri, Configuration conf) throws IOException {
     super(conf);
     kmsUrl = createServiceURL(extractKMSPath(uri));
+    int kmsPort = kmsUrl.getPort();
+    if ((kmsPort == -1) && fallbackDefaultPortForTesting) {
+      kmsPort = 9600;
+    }
+
+    InetSocketAddress addr = new InetSocketAddress(kmsUrl.getHost(), kmsPort);
+    dtService = SecurityUtil.buildTokenService(addr);
+
     if ("https".equalsIgnoreCase(kmsUrl.getProtocol())) {
       sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
       try {
@@ -386,9 +450,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
             CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_SECONDS,
             CommonConfigurationKeysPublic.KMS_CLIENT_TIMEOUT_DEFAULT);
     authRetry = conf.getInt(AUTH_RETRY, DEFAULT_AUTH_RETRY);
-    copyLegacyToken = conf.getBoolean(KMS_CLIENT_COPY_LEGACY_TOKEN_KEY,
-        KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT);
-
     configurator = new TimeoutConnConfigurator(timeout, sslFactory);
     encKeyVersionQueue =
         new ValueQueue<KeyProviderCryptoExtension.EncryptedKeyVersion>(
@@ -413,7 +474,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
                     KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
             new EncryptedQueueRefiller());
     authToken = new DelegationTokenAuthenticatedURL.Token();
-    dtService = new Text(providerUri.toString());
     LOG.info("KMSClientProvider for KMS url: {} delegation token service: {}" +
         " created.", kmsUrl, dtService);
   }
@@ -487,7 +547,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
         @Override
         public HttpURLConnection run() throws Exception {
           DelegationTokenAuthenticatedURL authUrl =
-              createKMSAuthenticatedURL();
+              new DelegationTokenAuthenticatedURL(configurator);
           return authUrl.openConnection(url, authToken, doAsUser);
         }
       });
@@ -865,7 +925,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
       LOG.debug("Renewing delegation token {} with url:{}, as:{}",
           token, url, doAsUser);
       final DelegationTokenAuthenticatedURL authUrl =
-          createKMSAuthenticatedURL();
+          new DelegationTokenAuthenticatedURL(configurator);
       return getActualUgi().doAs(
           new PrivilegedExceptionAction<Long>() {
             @Override
@@ -897,7 +957,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
               LOG.debug("Cancelling delegation token {} with url:{}, as:{}",
                   dToken, url, doAsUser);
               final DelegationTokenAuthenticatedURL authUrl =
-                  createKMSAuthenticatedURL();
+                  new DelegationTokenAuthenticatedURL(configurator);
               authUrl.cancelDelegationToken(url, token, doAsUser);
               return null;
             }
@@ -949,17 +1009,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     return token;
   }
 
-  @VisibleForTesting
-  DelegationTokenAuthenticatedURL createKMSAuthenticatedURL() {
-    return new DelegationTokenAuthenticatedURL(configurator) {
-      @Override
-      public org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
-          getDelegationToken(URL url, Credentials creds) {
-        return selectKMSDelegationToken(creds);
-      }
-    };
-  }
-
   @Override
   public Token<?>[] addDelegationTokens(final String renewer,
       Credentials credentials) throws IOException {
@@ -968,7 +1017,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     if (token == null) {
       final URL url = createURL(null, null, null, null);
       final DelegationTokenAuthenticatedURL authUrl =
-          createKMSAuthenticatedURL();
+          new DelegationTokenAuthenticatedURL(configurator);
       try {
         final String doAsUser = getDoAsUser();
         token = getActualUgi().doAs(new PrivilegedExceptionAction<Token<?>>() {
@@ -982,16 +1031,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
           }
         });
         if (token != null) {
-          if (KMSDelegationToken.TOKEN_KIND.equals(token.getKind())) {
-            // do not set service for legacy kind, for compatibility.
-            token.setService(dtService);
-          }
-          LOG.info("New token created: ({})", token);
+          LOG.debug("New token received: ({})", token);
           credentials.addToken(token.getService(), token);
-          Token<?> legacyToken = createAndAddLegacyToken(credentials, token);
-          tokens = legacyToken == null ?
-              new Token<?>[] {token} :
-              new Token<?>[] {token, legacyToken};
+          tokens = new Token<?>[] { token };
         } else {
           throw new IOException("Got NULL as delegation token");
         }
@@ -1008,75 +1050,13 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
     return tokens;
   }
 
-  /**
-   * If {@link CommonConfigurationKeysPublic#KMS_CLIENT_COPY_LEGACY_TOKEN_KEY}
-   * is true when creating the provider, then copy the passed-in token of
-   * {@link KMSDelegationToken#TOKEN_KIND} and create a new token of
-   * {@link KMSDelegationToken#TOKEN_LEGACY_KIND}, and add it to credentials.
-   *
-   * @return The legacy token, or null if one should not be created.
-   */
-  private Token<?> createAndAddLegacyToken(Credentials credentials,
-      Token<?> token) {
-    if (!copyLegacyToken || !KMSDelegationToken.TOKEN_KIND
-        .equals(token.getKind())) {
-      LOG.debug("Not creating legacy token because copyLegacyToken={}, "
-          + "token={}", copyLegacyToken, token);
-      return null;
-    }
-    // copy a KMS_DELEGATION_TOKEN and create a new kms-dt with the same
-    // underlying token for backwards-compatibility. Old clients/renewers
-    // does not parse the new token and can only work with kms-dt.
-    final Token<?> legacyToken = new Token(token);
-    legacyToken.setKind(KMSDelegationToken.TOKEN_LEGACY_KIND);
-    final InetSocketAddress addr =
-        new InetSocketAddress(kmsUrl.getHost(), kmsUrl.getPort());
-    final Text fallBackServiceText = SecurityUtil.buildTokenService(addr);
-    legacyToken.setService(fallBackServiceText);
-    LOG.info("Copied token to legacy kind: {}", legacyToken);
-    credentials.addToken(legacyToken.getService(), legacyToken);
-    return legacyToken;
-  }
-
-  @VisibleForTesting
-  public Text getDelegationTokenService() {
-    return dtService;
-  }
-
-  /**
-   * Given a list of tokens, return the token that should be used for KMS
-   * authentication.
-   */
-  @VisibleForTesting
-  Token selectKMSDelegationToken(Credentials creds) {
-    // always look for TOKEN_KIND first
-    final TokenSelector<AbstractDelegationTokenIdentifier> tokenSelector =
-        new AbstractDelegationTokenSelector<AbstractDelegationTokenIdentifier>(
-            KMSDelegationToken.TOKEN_KIND) {
-        };
-    Token token = tokenSelector.selectToken(dtService, creds.getAllTokens());
-    LOG.debug("Searching service {} found token {}", dtService, token);
-    if (token != null) {
-      return token;
-    }
-
-    // fall back to look for token by service, regardless of kind.
-    // this is old behavior, keeping for compatibility reasons (for example,
-    // even if KMS server is new, if the job is submitted with an old kms
-    // client, job runners on new version should be able to find the token).
-    final InetSocketAddress addr =
-        new InetSocketAddress(kmsUrl.getHost(), kmsUrl.getPort());
-    final Text fallBackServiceText = SecurityUtil.buildTokenService(addr);
-    token = creds.getToken(fallBackServiceText);
-    LOG.debug("Selected delegation token {} using service:{}", token,
-        fallBackServiceText);
-    return token;
-  }
-
-  private boolean containsKmsDt(UserGroupInformation ugi) {
-    final Credentials creds = ugi.getCredentials();
+  private boolean containsKmsDt(UserGroupInformation ugi) throws IOException {
+    // Add existing credentials from the UGI, since provider is cached.
+    Credentials creds = ugi.getCredentials();
     if (!creds.getAllTokens().isEmpty()) {
-      final Token dToken = selectKMSDelegationToken(creds);
+      LOG.debug("Searching for token that matches service: {}", dtService);
+      org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
+          dToken = creds.getToken(dtService);
       if (dToken != null) {
         return true;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
index 2642e79..adeebf2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSDelegationToken.java
@@ -27,10 +27,7 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier
 @InterfaceAudience.Private
 public final class KMSDelegationToken {
 
-  public static final String TOKEN_LEGACY_KIND_STR = "kms-dt";
-  public static final Text TOKEN_LEGACY_KIND = new Text(TOKEN_LEGACY_KIND_STR);
-
-  public static final String TOKEN_KIND_STR = "KMS_DELEGATION_TOKEN";
+  public static final String TOKEN_KIND_STR = "kms-dt";
   public static final Text TOKEN_KIND = new Text(TOKEN_KIND_STR);
 
   // Utility class is not supposed to be instantiated.
@@ -52,21 +49,4 @@ public final class KMSDelegationToken {
       return TOKEN_KIND;
     }
   }
-
-  /**
-   * DelegationTokenIdentifier used for the KMS for legacy tokens.
-   */
-  @Deprecated
-  public static class KMSLegacyDelegationTokenIdentifier
-      extends DelegationTokenIdentifier {
-
-    public KMSLegacyDelegationTokenIdentifier() {
-      super(TOKEN_LEGACY_KIND);
-    }
-
-    @Override
-    public Text getKind() {
-      return TOKEN_LEGACY_KIND;
-    }
-  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java
deleted file mode 100644
index fd27073..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSLegacyTokenRenewer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto.key.kms;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.key.KeyProvider;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.KMSUtil;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
-
-/**
- * The {@link KMSTokenRenewer} that supports legacy tokens.
- */
-@InterfaceAudience.Private
-@Deprecated
-public class KMSLegacyTokenRenewer extends KMSTokenRenewer {
-
-  @Override
-  public boolean handleKind(Text kind) {
-    return kind.equals(TOKEN_LEGACY_KIND);
-  }
-
-  /**
-   * Create a key provider for token renewal / cancellation.
-   * Caller is responsible for closing the key provider.
-   */
-  @Override
-  protected KeyProvider createKeyProvider(Token<?> token,
-      Configuration conf) throws IOException {
-    assert token.getKind().equals(TOKEN_LEGACY_KIND);
-    // Legacy tokens get service from configuration.
-    return KMSUtil.createKeyProvider(conf,
-        CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java
deleted file mode 100644
index 1fff2f9..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSTokenRenewer.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto.key.kms;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.key.KeyProvider;
-import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenRenewer;
-import org.apache.hadoop.util.KMSUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
-
-/**
- * The KMS implementation of {@link TokenRenewer}.
- */
-@InterfaceAudience.Private
-public class KMSTokenRenewer extends TokenRenewer {
-
-  public static final Logger LOG = LoggerFactory
-      .getLogger(org.apache.hadoop.crypto.key.kms.KMSTokenRenewer.class);
-
-  @Override
-  public boolean handleKind(Text kind) {
-    return kind.equals(TOKEN_KIND);
-  }
-
-  @Override
-  public boolean isManaged(Token<?> token) throws IOException {
-    return true;
-  }
-
-  @Override
-  public long renew(Token<?> token, Configuration conf) throws IOException {
-    LOG.debug("Renewing delegation token {}", token);
-    final KeyProvider keyProvider = createKeyProvider(token, conf);
-    try {
-      if (!(keyProvider instanceof
-          KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-        throw new IOException(String
-            .format("keyProvider %s cannot renew token [%s]",
-                keyProvider == null ? "null" : keyProvider.getClass(), token));
-      }
-      return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-          keyProvider).renewDelegationToken(token);
-    } finally {
-      if (keyProvider != null) {
-        keyProvider.close();
-      }
-    }
-  }
-
-  @Override
-  public void cancel(Token<?> token, Configuration conf) throws IOException {
-    LOG.debug("Canceling delegation token {}", token);
-    final KeyProvider keyProvider = createKeyProvider(token, conf);
-    try {
-      if (!(keyProvider instanceof
-          KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-        throw new IOException(String
-            .format("keyProvider %s cannot cancel token [%s]",
-                keyProvider == null ? "null" : keyProvider.getClass(), token));
-      }
-      ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-          keyProvider).cancelDelegationToken(token);
-    } finally {
-      if (keyProvider != null) {
-        keyProvider.close();
-      }
-    }
-  }
-
-  /**
-   * Create a key provider for token renewal / cancellation.
-   * Caller is responsible for closing the key provider.
-   */
-  protected KeyProvider createKeyProvider(Token<?> token,
-      Configuration conf) throws IOException {
-    return KMSUtil
-        .createKeyProviderFromTokenService(conf, token.getService().toString());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java
deleted file mode 100644
index eea93c2..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto.key.kms;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 5fbaef4..c8c1c52 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -721,16 +721,6 @@ public class CommonConfigurationKeysPublic {
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
    * core-default.xml</a>
    */
-  public static final String KMS_CLIENT_COPY_LEGACY_TOKEN_KEY =
-      "hadoop.security.kms.client.copy.legacy.token";
-  /**  Default value is true. */
-  public static final boolean KMS_CLIENT_COPY_LEGACY_TOKEN_DEFAULT = true;
-
-  /**
-   * @see
-   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
-   * core-default.xml</a>
-   */
   public static final String KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY =
       "hadoop.security.kms.client.failover.sleep.max.millis";
   /** Default value is 2 secs. */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
index 0ddc4fc..0b1fdf8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticatedURL.java
@@ -300,7 +300,11 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
             creds.getAllTokens());
       }
       if (!creds.getAllTokens().isEmpty()) {
-        dToken = getDelegationToken(url, creds);
+        InetSocketAddress serviceAddr = new InetSocketAddress(url.getHost(),
+            url.getPort());
+        Text service = SecurityUtil.buildTokenService(serviceAddr);
+        dToken = creds.getToken(service);
+        LOG.debug("Using delegation token {} from service:{}", dToken, service);
         if (dToken != null) {
           if (useQueryStringForDelegationToken()) {
             // delegation token will go in the query string, injecting it
@@ -337,21 +341,6 @@ public class DelegationTokenAuthenticatedURL extends AuthenticatedURL {
   }
 
   /**
-   * Select a delegation token from all tokens in credentials, based on url.
-   */
-  @InterfaceAudience.Private
-  public org.apache.hadoop.security.token.Token<? extends TokenIdentifier>
-      getDelegationToken(URL url, Credentials creds) {
-    final InetSocketAddress serviceAddr =
-        new InetSocketAddress(url.getHost(), url.getPort());
-    final Text service = SecurityUtil.buildTokenService(serviceAddr);
-    org.apache.hadoop.security.token.Token<? extends TokenIdentifier> dToken =
-        creds.getToken(service);
-    LOG.debug("Selected delegation token {} using service:{}", dToken, service);
-    return dToken;
-  }
-
-  /**
    * Requests a delegation token using the configured <code>Authenticator</code>
    * for authentication.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index f66477c..a53472f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -79,7 +79,7 @@ import com.google.common.annotations.VisibleForTesting;
 @InterfaceStability.Evolving
 public abstract class DelegationTokenAuthenticationHandler
     implements AuthenticationHandler {
-  public static final Logger LOG =
+  private static final Logger LOG =
       LoggerFactory.getLogger(DelegationTokenAuthenticationHandler.class);
 
   protected static final String TYPE_POSTFIX = "-dt";
@@ -194,8 +194,7 @@ public abstract class DelegationTokenAuthenticationHandler
       HttpServletRequest request, HttpServletResponse response)
       throws IOException, AuthenticationException {
     boolean requestContinues = true;
-    LOG.trace("Processing operation for req=({}), token: {}",
-        request.getRequestURL(), token);
+    LOG.trace("Processing operation for req=({}), token: {}", request, token);
     String op = ServletUtils.getParameter(request,
         KerberosDelegationTokenAuthenticator.OP_PARAM);
     op = (op != null) ? StringUtils.toUpperCase(op) : null;
@@ -378,8 +377,7 @@ public abstract class DelegationTokenAuthenticationHandler
             HttpServletResponse.SC_FORBIDDEN, new AuthenticationException(ex));
       }
     } else {
-      LOG.debug("Falling back to {} (req={})", authHandler.getClass(),
-          request.getRequestURL());
+      LOG.debug("Falling back to {} (req={})", authHandler.getClass(), request);
       token = authHandler.authenticate(request, response);
     }
     return token;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 4f144c5..8f17e6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -48,7 +48,7 @@ import java.util.Map;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public abstract class DelegationTokenAuthenticator implements Authenticator {
-  public static final Logger LOG =
+  private static Logger LOG = 
       LoggerFactory.getLogger(DelegationTokenAuthenticator.class);
   
   private static final String CONTENT_TYPE = "Content-Type";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
index f70585c..5f783a9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
@@ -26,14 +26,14 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.net.URI;
-import java.net.URISyntaxException;
 
 /**
  * Utils for KMS.
  */
 @InterfaceAudience.Private
 public final class KMSUtil {
-  public static final Logger LOG = LoggerFactory.getLogger(KMSUtil.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(KMSUtil.class);
 
   private KMSUtil() { /* Hidden constructor */ }
 
@@ -55,13 +55,6 @@ public final class KMSUtil {
     if (providerUriStr == null || providerUriStr.isEmpty()) {
       return null;
     }
-    KeyProvider kp = KMSUtilFaultInjector.get().createKeyProviderForTests(
-        providerUriStr, conf);
-    if (kp != null) {
-      LOG.info("KeyProvider is created with uri: {}. This should happen only " +
-              "in tests.", providerUriStr);
-      return kp;
-    }
     return createKeyProviderFromUri(conf, URI.create(providerUriStr));
   }
 
@@ -78,38 +71,4 @@ public final class KMSUtil {
     }
     return keyProvider;
   }
-
-  /**
-   * Creates a key provider from token service field, which must be URI format.
-   *
-   * @param conf
-   * @param tokenServiceValue
-   * @return new KeyProvider or null
-   * @throws IOException
-   */
-  public static KeyProvider createKeyProviderFromTokenService(
-      final Configuration conf, final String tokenServiceValue)
-      throws IOException {
-    LOG.debug("Creating key provider from token service value {}. ",
-        tokenServiceValue);
-    final KeyProvider kp = KMSUtilFaultInjector.get()
-        .createKeyProviderForTests(tokenServiceValue, conf);
-    if (kp != null) {
-      LOG.info("KeyProvider is created with uri: {}. This should happen only "
-          + "in tests.", tokenServiceValue);
-      return kp;
-    }
-    if (!tokenServiceValue.contains("://")) {
-      throw new IllegalArgumentException(
-          "Invalid token service " + tokenServiceValue);
-    }
-    final URI tokenServiceUri;
-    try {
-      tokenServiceUri = new URI(tokenServiceValue);
-    } catch (URISyntaxException e) {
-      throw new IllegalArgumentException(
-          "Invalid token service " + tokenServiceValue, e);
-    }
-    return createKeyProviderFromUri(conf, tokenServiceUri);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java
deleted file mode 100644
index 46d5069..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtilFaultInjector.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.util;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.key.KeyProvider;
-
-import java.io.IOException;
-
-/**
- * Used for returning custom KeyProvider from test methods.
- */
-@VisibleForTesting
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class KMSUtilFaultInjector {
-  private static KMSUtilFaultInjector instance = new KMSUtilFaultInjector();
-
-  public static KMSUtilFaultInjector get() {
-    return instance;
-  }
-
-  public static void set(KMSUtilFaultInjector injector) {
-    instance = injector;
-  }
-
-  public KeyProvider createKeyProviderForTests(String value, Configuration conf)
-      throws IOException {
-    return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 43d06e2..b65f151 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -12,4 +12,3 @@
 #   limitations under the License.
 #
 org.apache.hadoop.crypto.key.kms.KMSDelegationToken$KMSDelegationTokenIdentifier
-org.apache.hadoop.crypto.key.kms.KMSDelegationToken$KMSLegacyDelegationTokenIdentifier

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
index 5b6082c..56320fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -11,5 +11,4 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 #
-org.apache.hadoop.crypto.key.kms.KMSTokenRenewer
-org.apache.hadoop.crypto.key.kms.KMSLegacyTokenRenewer
\ No newline at end of file
+org.apache.hadoop.crypto.key.kms.KMSClientProvider$KMSTokenRenewer
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 9d4af62..aa041a0 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2183,26 +2183,6 @@
 </property>
 
 <property>
-  <name>hadoop.security.kms.client.copy.legacy.token</name>
-  <value>true</value>
-  <description>
-    Expert only. Whether the KMS client provider should copy a token to legacy
-    kind. This is for KMS_DELEGATION_TOKEN to be backwards compatible. With the
-    default value set to true, the client will locally duplicate the
-    KMS_DELEGATION_TOKEN token and create a kms-dt token, with the service field
-    conforming to kms-dt. All other parts of the token remain the same.
-    Then the new clients will use KMS_DELEGATION_TOKEN and old clients will
-    use kms-dt to authenticate. Default value is true.
-    You should only change this to false if you know all the KMS servers
-    , clients (including both job submitters and job runners) and the
-    token renewers (usually Yarn RM) are on a version that supports
-    KMS_DELEGATION_TOKEN.
-    Turning this off prematurely may result in old clients failing to
-    authenticate with new servers.
-  </description>
-</property>
-
-<property>
   <name>hadoop.security.kms.client.failover.sleep.max.millis</name>
   <value>2000</value>
   <description>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java
deleted file mode 100644
index f1b7bbd..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestKMSClientProvider.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto.key.kms;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.net.URL;
-
-import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_KIND;
-import static org.apache.hadoop.crypto.key.kms.KMSDelegationToken.TOKEN_LEGACY_KIND;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-/**
- * Unit test for {@link KMSClientProvider} class.
- */
-public class TestKMSClientProvider {
-
-  public static final Logger LOG =
-      LoggerFactory.getLogger(TestKMSClientProvider.class);
-
-  private final Token token = new Token();
-  private final Token legacyToken = new Token();
-  private final String uriString = "kms://https@host:16000/kms";
-  private final String legacyTokenService = "host:16000";
-
-  @Rule
-  public Timeout globalTimeout = new Timeout(30000);
-
-  {
-    GenericTestUtils.setLogLevel(KMSClientProvider.LOG, Level.TRACE);
-  }
-
-  @Before
-  public void setup() {
-    SecurityUtil.setTokenServiceUseIp(false);
-    token.setKind(TOKEN_KIND);
-    token.setService(new Text(uriString));
-    legacyToken.setKind(TOKEN_LEGACY_KIND);
-    legacyToken.setService(new Text(legacyTokenService));
-  }
-
-  @SuppressWarnings("unchecked")
-  @Test
-  public void testNotCopyFromLegacyToken() throws Exception {
-    final DelegationTokenAuthenticatedURL url =
-        mock(DelegationTokenAuthenticatedURL.class);
-    final Configuration conf = new Configuration();
-    final URI uri = new URI(uriString);
-    final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
-    try {
-      final KMSClientProvider spyKp = spy(kp);
-      when(spyKp.createKMSAuthenticatedURL()).thenReturn(url);
-      when(url.getDelegationToken(any(URL.class),
-          any(DelegationTokenAuthenticatedURL.Token.class), any(String.class),
-          any(String.class))).thenReturn(legacyToken);
-
-      final Credentials creds = new Credentials();
-      final Token<?>[] tokens = spyKp.addDelegationTokens("yarn", creds);
-      LOG.info("Got tokens: {}", tokens);
-      assertEquals(1, tokens.length);
-      LOG.info("uri:" + uriString);
-      // if KMS server returned a legacy token, new client should leave the
-      // service being legacy and not set uri string
-      assertEquals(legacyTokenService, tokens[0].getService().toString());
-    } finally {
-      kp.close();
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  @Test
-  public void testCopyFromToken() throws Exception {
-    final DelegationTokenAuthenticatedURL url =
-        mock(DelegationTokenAuthenticatedURL.class);
-    final Configuration conf = new Configuration();
-    final URI uri = new URI(uriString);
-    final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
-    try {
-      final KMSClientProvider spyKp = spy(kp);
-      when(spyKp.createKMSAuthenticatedURL()).thenReturn(url);
-      when(url.getDelegationToken(any(URL.class),
-          any(DelegationTokenAuthenticatedURL.Token.class), any(String.class),
-          any(String.class))).thenReturn(token);
-
-      final Credentials creds = new Credentials();
-      final Token<?>[] tokens = spyKp.addDelegationTokens("yarn", creds);
-      LOG.info("Got tokens: {}", tokens);
-      assertEquals(2, tokens.length);
-      assertTrue(creds.getAllTokens().contains(token));
-      assertNotNull(creds.getToken(legacyToken.getService()));
-    } finally {
-      kp.close();
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  @Test
-  public void testSelectTokenWhenBothExist() throws Exception {
-    final Credentials creds = new Credentials();
-    final Configuration conf = new Configuration();
-    final URI uri = new URI(uriString);
-    final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
-    try {
-      creds.addToken(token.getService(), token);
-      creds.addToken(legacyToken.getService(), legacyToken);
-      Token t = kp.selectKMSDelegationToken(creds);
-      assertEquals(token, t);
-    } finally {
-      kp.close();
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  @Test
-  public void testSelectTokenLegacyService() throws Exception {
-    final Configuration conf = new Configuration();
-    final URI uri = new URI(uriString);
-    final KMSClientProvider kp = new KMSClientProvider(uri, conf, uri);
-    try {
-      Text legacyService = new Text(legacyTokenService);
-      token.setService(legacyService);
-      final Credentials creds = new Credentials();
-      creds.addToken(legacyService, token);
-      Token t = kp.selectKMSDelegationToken(creds);
-      assertEquals(token, t);
-    } finally {
-      kp.close();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
index 7c4c158..0270d26 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
@@ -42,8 +42,7 @@ import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;
 import org.apache.hadoop.security.authorize.AuthorizationException;
-import org.apache.hadoop.util.KMSUtil;
-import org.apache.hadoop.util.KMSUtilFaultInjector;
+import org.junit.After;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -57,68 +56,33 @@ public class TestLoadBalancingKMSClientProvider {
     SecurityUtil.setTokenServiceUseIp(false);
   }
 
-  private void setKMSUtilFaultInjector() {
-    KMSUtilFaultInjector injector = new KMSUtilFaultInjector() {
-      @Override
-      public KeyProvider createKeyProviderForTests(
-          String value, Configuration conf) throws IOException {
-        return TestLoadBalancingKMSClientProvider
-            .createKeyProviderForTests(value, conf);
-      }
-    };
-    KMSUtilFaultInjector.set(injector);
-  }
-
-  public static KeyProvider createKeyProviderForTests(
-      String value, Configuration conf) throws IOException {
-    // The syntax for kms servers will be
-    // kms://http@localhost:port1/kms,kms://http@localhost:port2/kms
-    if (!value.contains(",")) {
-      return null;
-    }
-    String[] keyProviderUrisStr = value.split(",");
-    KMSClientProvider[] keyProviderArr =
-        new KMSClientProvider[keyProviderUrisStr.length];
-
-    int i = 0;
-    for (String keyProviderUri: keyProviderUrisStr) {
-      KMSClientProvider kmcp =
-          new KMSClientProvider(URI.create(keyProviderUri), conf, URI
-              .create(value));
-      keyProviderArr[i] = kmcp;
-      i++;
-    }
-    LoadBalancingKMSClientProvider lbkcp =
-        new LoadBalancingKMSClientProvider(keyProviderArr, conf);
-    return lbkcp;
+  @After
+  public void teardown() throws IOException {
+    KMSClientProvider.fallbackDefaultPortForTesting = false;
   }
 
   @Test
   public void testCreation() throws Exception {
     Configuration conf = new Configuration();
+    KMSClientProvider.fallbackDefaultPortForTesting = true;
     KeyProvider kp = new KMSClientProvider.Factory().createProvider(new URI(
-        "kms://http@host1:9600/kms/foo"), conf);
+        "kms://http@host1/kms/foo"), conf);
     assertTrue(kp instanceof LoadBalancingKMSClientProvider);
     KMSClientProvider[] providers =
         ((LoadBalancingKMSClientProvider) kp).getProviders();
     assertEquals(1, providers.length);
-    assertEquals(Sets.newHashSet("http://host1:9600/kms/foo/v1/"),
+    assertEquals(Sets.newHashSet("http://host1/kms/foo/v1/"),
         Sets.newHashSet(providers[0].getKMSUrl()));
-    setKMSUtilFaultInjector();
-    String uriStr = "kms://http@host1:9600/kms/foo," +
-        "kms://http@host2:9600/kms/foo," +
-        "kms://http@host3:9600/kms/foo";
-    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
-        uriStr);
-    kp = KMSUtil.createKeyProvider(conf, CommonConfigurationKeysPublic
-        .HADOOP_SECURITY_KEY_PROVIDER_PATH);
+
+    kp = new KMSClientProvider.Factory().createProvider(new URI(
+        "kms://http@host1;host2;host3/kms/foo"), conf);
     assertTrue(kp instanceof LoadBalancingKMSClientProvider);
     providers =
         ((LoadBalancingKMSClientProvider) kp).getProviders();
     assertEquals(3, providers.length);
-    assertEquals(Sets.newHashSet("http://host1:9600/kms/foo/v1/",
-        "http://host2:9600/kms/foo/v1/",
-        "http://host3:9600/kms/foo/v1/"),
+    assertEquals(Sets.newHashSet("http://host1/kms/foo/v1/",
+        "http://host2/kms/foo/v1/",
+        "http://host3/kms/foo/v1/"),
         Sets.newHashSet(providers[0].getKMSUrl(),
             providers[1].getKMSUrl(),
             providers[2].getKMSUrl()));
@@ -244,7 +208,7 @@ public class TestLoadBalancingKMSClientProvider {
 
   private class MyKMSClientProvider extends KMSClientProvider {
     public MyKMSClientProvider(URI uri, Configuration conf) throws IOException {
-      super(uri, conf, uri);
+      super(uri, conf);
     }
 
     @Override
@@ -281,8 +245,9 @@ public class TestLoadBalancingKMSClientProvider {
   @Test
   public void testClassCastException() throws Exception {
     Configuration conf = new Configuration();
+    KMSClientProvider.fallbackDefaultPortForTesting = true;
     KMSClientProvider p1 = new MyKMSClientProvider(
-        new URI("kms://http@host1:9600/kms/foo"), conf);
+        new URI("kms://http@host1/kms/foo"), conf);
     LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
         new KMSClientProvider[] {p1}, 0, conf);
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d10c4a97/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java
deleted file mode 100644
index 77f52ee..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestKMSUtil.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.key.KeyProvider;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test {@link KMSUtil}.
- */
-public class TestKMSUtil {
-
-  public static final Logger LOG = LoggerFactory.getLogger(TestKMSUtil.class);
-
-  @Rule
-  public Timeout globalTimeout = new Timeout(90000);
-
-  @Test
-  public void testCreateKeyProviderFromTokenService() throws Exception {
-    final Configuration conf = new Configuration();
-    KeyProvider kp = KMSUtil.createKeyProviderFromTokenService(conf,
-        "kms://https@localhost:9600/kms");
-    assertNotNull(kp);
-    kp.close();
-
-    kp = KMSUtil.createKeyProviderFromTokenService(conf,
-        "kms://https@localhost:9600/kms,kms://localhost1:9600/kms");
-    assertNotNull(kp);
-    kp.close();
-
-    String invalidService = "whatever:9600";
-    try {
-      KMSUtil.createKeyProviderFromTokenService(conf, invalidService);
-    } catch (Exception ex) {
-      LOG.info("Expected exception:", ex);
-      assertTrue(ex instanceof IllegalArgumentException);
-      GenericTestUtils.assertExceptionContains(
-          "Invalid token service " + invalidService, ex);
-    }
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message