Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 07F13200D29 for ; Thu, 12 Oct 2017 01:03:36 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 06665160BE4; Wed, 11 Oct 2017 23:03:36 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 76AF4160BF1 for ; Thu, 12 Oct 2017 01:03:32 +0200 (CEST) Received: (qmail 51871 invoked by uid 500); 11 Oct 2017 23:03:31 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 50662 invoked by uid 99); 11 Oct 2017 23:03:30 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 11 Oct 2017 23:03:30 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 09362DFCC2; Wed, 11 Oct 2017 23:03:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jaimin@apache.org To: commits@ambari.apache.org Date: Wed, 11 Oct 2017 23:03:59 -0000 Message-Id: <59a1489c027f4b20a72455064fb09a2f@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [32/50] [abbrv] ambari git commit: AMBARI-22138. When regenerating keytab files for a service, non-service-specific principals are affected (rlevas) archived-at: Wed, 11 Oct 2017 23:03:36 -0000 AMBARI-22138. When regenerating keytab files for a service, non-service-specific principals are affected (rlevas) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5af1e539 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5af1e539 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5af1e539 Branch: refs/heads/branch-feature-AMBARI-14714-ui Commit: 5af1e539cce928b32fc5aca67c7bf8dbc2bd3c2e Parents: b0c24a5 Author: Robert Levas Authored: Mon Oct 9 13:06:13 2017 -0400 Committer: Robert Levas Committed: Mon Oct 9 13:06:19 2017 -0400 ---------------------------------------------------------------------- .../server/controller/KerberosHelper.java | 9 +- .../server/controller/KerberosHelperImpl.java | 244 ++++++++++++------- .../utilities/RemovableIdentities.java | 2 +- .../kerberos/CreateKeytabFilesServerAction.java | 2 +- .../kerberos/CreatePrincipalsServerAction.java | 2 +- .../kerberos/KerberosServerAction.java | 71 ++++-- .../PrepareDisableKerberosServerAction.java | 3 +- .../PrepareEnableKerberosServerAction.java | 6 +- .../PrepareKerberosIdentitiesServerAction.java | 142 ++++++++--- .../kerberos/AbstractKerberosDescriptor.java | 25 ++ .../AbstractKerberosDescriptorContainer.java | 18 +- .../kerberos/KerberosIdentityDescriptor.java | 160 ++++++++++++ .../server/controller/KerberosHelperTest.java | 5 - .../utilities/KerberosIdentityCleanerTest.java | 8 +- .../state/kerberos/KerberosDescriptorTest.java | 150 +++++++++++- 15 files changed, 665 insertions(+), 182 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java index 20c5708..b8e1be1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java @@ -442,12 +442,6 @@ public interface KerberosHelper { * @param hostFilter a set of hostname indicating the set of hosts to process - * if null, no filter is relevant; if empty, the filter * indicates no relevant hosts - * @param identityFilter a Collection of identity names indicating the relevant - * identities - if null, no filter is relevant; if empty, - * the filter indicates no relevant identities - * @param shouldProcessCommand a Command implementation to determine if the relevant component - * is in a state in which is should be process for the current - * Kerberos operation. * @return a list of ServiceComponentHost instances and should be processed during the relevant * Kerberos operation. * @throws AmbariException @@ -455,8 +449,7 @@ public interface KerberosHelper { List getServiceComponentHostsToProcess(Cluster cluster, KerberosDescriptor kerberosDescriptor, Map> serviceComponentFilter, - Collection hostFilter, Collection identityFilter, - Command shouldProcessCommand) + Collection hostFilter) throws AmbariException; Set getHostsWithValidKerberosClient(Cluster cluster) throws AmbariException; http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java index b691968..f8fe31a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java @@ -122,6 +122,7 @@ import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; import org.apache.ambari.server.state.kerberos.VariableReplacementHelper; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostServerActionEvent; import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.directory.server.kerberos.shared.keytab.Keytab; @@ -268,10 +269,12 @@ public class KerberosHelperImpl implements KerberosHelper { boolean updateConfigurations = !requestProperties.containsKey(DIRECTIVE_IGNORE_CONFIGS) || !"true".equalsIgnoreCase(requestProperties.get(DIRECTIVE_IGNORE_CONFIGS)); + boolean forceAllHosts = (hostFilter == null) || (hostFilter.contains("*")); + if ("true".equalsIgnoreCase(value) || "all".equalsIgnoreCase(value)) { - handler = new CreatePrincipalsAndKeytabsHandler(true, updateConfigurations, true); + handler = new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.RECREATE_ALL, updateConfigurations, forceAllHosts, true); } else if ("missing".equalsIgnoreCase(value)) { - handler = new CreatePrincipalsAndKeytabsHandler(false, updateConfigurations, true); + handler = new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.CREATE_MISSING, updateConfigurations, forceAllHosts, true); } if (handler != null) { @@ -326,7 +329,7 @@ public class KerberosHelperImpl implements KerberosHelper { if (serviceComponentsArray.length == 2) { serviceComponentFilter.put(serviceName, ImmutableSet.copyOf(serviceComponentsArray[1].split(";"))); } else { - serviceComponentFilter.put(serviceName, null); + serviceComponentFilter.put(serviceName, ImmutableSet.of("*")); } } return serviceComponentFilter.build(); @@ -340,7 +343,7 @@ public class KerberosHelperImpl implements KerberosHelper { RequestStageContainer requestStageContainer, Boolean manageIdentities) throws AmbariException, KerberosOperationException { return handle(cluster, getKerberosDetails(cluster, manageIdentities), serviceComponentFilter, hostFilter, identityFilter, - hostsToForceKerberosOperations, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(false, false, + hostsToForceKerberosOperations, requestStageContainer, new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.DEFAULT, false, false, false)); } @@ -1061,7 +1064,7 @@ public class KerberosHelperImpl implements KerberosHelper { RequestStageContainer requestStageContainer) throws KerberosOperationException, AmbariException { return handleTestIdentity(cluster, getKerberosDetails(cluster, null), commandParamsStage, requestStageContainer, - new CreatePrincipalsAndKeytabsHandler(false, false, false)); + new CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType.DEFAULT, false, false, false)); } @Override @@ -1230,27 +1233,25 @@ public class KerberosHelperImpl implements KerberosHelper { public List getServiceComponentHostsToProcess(final Cluster cluster, final KerberosDescriptor kerberosDescriptor, final Map> serviceComponentFilter, - final Collection hostFilter, Collection identityFilter, - final Command shouldProcessCommand) + final Collection hostFilter) throws AmbariException { return getServiceComponentHosts(cluster, new Command() { @Override public Boolean invoke(ServiceComponentHost sch) throws AmbariException { if (sch != null) { // Check the host filter - if ((hostFilter == null) || hostFilter.contains(sch.getHostName())) { + if ((hostFilter == null) || hostFilter.contains("*") || hostFilter.contains(sch.getHostName())) { String serviceName = sch.getServiceName(); // Check the service filter - if ((serviceComponentFilter == null) || serviceComponentFilter.containsKey(serviceName)) { + if ((serviceComponentFilter == null) || serviceComponentFilter.containsKey("*") || serviceComponentFilter.containsKey(serviceName)) { KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(serviceName); if (serviceDescriptor != null) { - Collection componentFilter = (serviceComponentFilter == null) ? null : serviceComponentFilter.get(serviceName); + Collection componentFilter = ((serviceComponentFilter == null) || serviceComponentFilter.containsKey("*")) ? null : serviceComponentFilter.get(serviceName); - // Check the service/component filter and the shouldProcessCommand - return (((componentFilter == null) || componentFilter.contains(sch.getServiceComponentName())) && - ((shouldProcessCommand == null) || shouldProcessCommand.invoke(sch))); + // Check the service/component filter + return (((componentFilter == null) || componentFilter.contains("*") || componentFilter.contains(sch.getServiceComponentName()))); } } } @@ -1491,8 +1492,9 @@ public class KerberosHelperImpl implements KerberosHelper { if (identities != null) { for (KerberosIdentityDescriptor identity : identities) { - // If there is no filter or the filter contains the current identity's name... - if ((identityFilter == null) || identityFilter.contains(identity.getName())) { + // If there is no filter or the filter contains the current identity's path... + if ((identityFilter == null) || identityFilter.contains(identity.getPath())) { + KerberosPrincipalDescriptor principalDescriptor = identity.getPrincipalDescriptor(); String principal = null; String principalType = null; @@ -2030,10 +2032,7 @@ public class KerberosHelperImpl implements KerberosHelper { cluster, kerberosDescriptor, serviceComponentFilter, - hostFilter, - identityFilter, - arg -> true); - + hostFilter); // While iterating over all the ServiceComponentHosts find hosts that have KERBEROS_CLIENT // components in the INSTALLED state and add them to the hostsWithValidKerberosClient Set. @@ -3378,12 +3377,11 @@ public class KerberosHelperImpl implements KerberosHelper { requestStageContainer.addStages(roleGraph.getStages()); } - public void addDistributeKeytabFilesStage(Cluster cluster, List serviceComponentHosts, - String clusterHostInfoJson, String hostParamsJson, - Map commandParameters, - RoleCommandOrder roleCommandOrder, - RequestStageContainer requestStageContainer, - Set hostsWithValidKerberosClient) + void addDistributeKeytabFilesStage(Cluster cluster, String clusterHostInfoJson, + String hostParamsJson, Map commandParameters, + RoleCommandOrder roleCommandOrder, + RequestStageContainer requestStageContainer, + List hosts) throws AmbariException { Stage stage = createNewStage(requestStageContainer.getLastStageId(), @@ -3393,20 +3391,13 @@ public class KerberosHelperImpl implements KerberosHelper { StageUtils.getGson().toJson(commandParameters), hostParamsJson); - Collection filteredComponents = filterServiceComponentHostsForHosts( - new ArrayList<>(serviceComponentHosts), hostsWithValidKerberosClient); - - if (!filteredComponents.isEmpty()) { - List hostsToUpdate = createUniqueHostList(filteredComponents, Collections.singleton(HostState.HEALTHY)); + if (!hosts.isEmpty()) { Map requestParams = new HashMap<>(); - List requestResourceFilters = new ArrayList<>(); - RequestResourceFilter reqResFilter = new RequestResourceFilter(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name(), hostsToUpdate); - requestResourceFilters.add(reqResFilter); ActionExecutionContext actionExecContext = new ActionExecutionContext( cluster.getClusterName(), SET_KEYTAB, - requestResourceFilters, + createRequestResourceFilters(hosts), requestParams); customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, requestParams, null); @@ -3422,7 +3413,12 @@ public class KerberosHelperImpl implements KerberosHelper { /** * Send a custom command to the KERBEROS_CLIENT to check if there are missing keytabs on each hosts. */ - public void addCheckMissingKeytabsStage(Cluster cluster, String clusterHostInfoJson, String hostParamsJson, ServiceComponentHostServerActionEvent event, Map commandParameters, RoleCommandOrder roleCommandOrder, RequestStageContainer requestStageContainer, List serviceComponentHosts) throws AmbariException { + void addCheckMissingKeytabsStage(Cluster cluster, String clusterHostInfoJson, + String hostParamsJson, Map commandParameters, + RoleCommandOrder roleCommandOrder, + RequestStageContainer requestStageContainer, + List hostsToInclude) + throws AmbariException { Stage stage = createNewStage(requestStageContainer.getLastStageId(), cluster, requestStageContainer.getId(), @@ -3430,20 +3426,13 @@ public class KerberosHelperImpl implements KerberosHelper { StageUtils.getGson().toJson(commandParameters), hostParamsJson); - Collection filteredComponents = filterServiceComponentHostsForHosts( - new ArrayList<>(serviceComponentHosts), getHostsWithValidKerberosClient(cluster)); - - if (!filteredComponents.isEmpty()) { - List hostsToUpdate = createUniqueHostList(filteredComponents, Collections.singleton(HostState.HEALTHY)); + if (!hostsToInclude.isEmpty()) { Map requestParams = new HashMap<>(); - List requestResourceFilters = new ArrayList<>(); - RequestResourceFilter reqResFilter = new RequestResourceFilter(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name(), hostsToUpdate); - requestResourceFilters.add(reqResFilter); ActionExecutionContext actionExecContext = new ActionExecutionContext( cluster.getClusterName(), CHECK_KEYTABS, - requestResourceFilters, + createRequestResourceFilters(hostsToInclude), requestParams); customCommandExecutionHelper.addExecutionCommandsToStage(actionExecContext, stage, requestParams, null); } @@ -3454,32 +3443,6 @@ public class KerberosHelperImpl implements KerberosHelper { requestStageContainer.addStages(roleGraph.getStages()); } - /** - * Filter out ServiceComponentHosts that are on on hosts in the specified set of host names. - *

- * It is expected that the supplied collection is modifiable. It will be modified inplace. - * - * @param serviceComponentHosts a collection of ServiceComponentHost items to test - * @param hosts a set of host names indicating valid hosts - * @return a collection of filtered ServiceComponentHost items - */ - private Collection filterServiceComponentHostsForHosts(Collection serviceComponentHosts, - Set hosts) { - - if ((serviceComponentHosts != null) && (hosts != null)) { - Iterator iterator = serviceComponentHosts.iterator(); - while (iterator.hasNext()) { - ServiceComponentHost sch = iterator.next(); - - if (!hosts.contains(sch.getHostName())) { - iterator.remove(); - } - } - } - - return serviceComponentHosts; - } - void addDisableSecurityHookStage(Cluster cluster, String clusterHostInfoJson, String hostParamsJson, @@ -3677,6 +3640,13 @@ public class KerberosHelperImpl implements KerberosHelper { requestStageContainer.setClusterHostInfo(clusterHostInfoJson); requestStageContainer.addStages(roleGraph.getStages()); } + + private List createRequestResourceFilters(List hostsToInclude) { + List requestResourceFilters = new ArrayList<>(); + RequestResourceFilter reqResFilter = new RequestResourceFilter(Service.Type.KERBEROS.name(), Role.KERBEROS_CLIENT.name(), hostsToInclude); + requestResourceFilters.add(reqResFilter); + return requestResourceFilters; + } } /** @@ -3746,6 +3716,8 @@ public class KerberosHelperImpl implements KerberosHelper { roleCommandOrder, requestStageContainer); if (kerberosDetails.manageIdentities()) { + List hostsToInclude = calculateHosts(cluster, serviceComponentHosts, hostsWithValidKerberosClient, false); + commandParameters.put(KerberosServerAction.KDC_TYPE, kerberosDetails.getKdcType().name()); // ***************************************************************** @@ -3767,8 +3739,8 @@ public class KerberosHelperImpl implements KerberosHelper { // ***************************************************************** // Create stage to distribute keytabs - addDistributeKeytabFilesStage(cluster, serviceComponentHosts, clusterHostInfoJson, hostParamsJson, - commandParameters, roleCommandOrder, requestStageContainer, hostsWithValidKerberosClient); + addDistributeKeytabFilesStage(cluster, clusterHostInfoJson, hostParamsJson, commandParameters, + roleCommandOrder, requestStageContainer, hostsToInclude); } // ***************************************************************** @@ -3885,10 +3857,11 @@ public class KerberosHelperImpl implements KerberosHelper { */ private class CreatePrincipalsAndKeytabsHandler extends Handler { /** - * A boolean value indicating whether to create keytabs for all principals (true) - * or only the ones that are missing (false). + * The type of Kerberos operation being performed. + * + * @see org.apache.ambari.server.serveraction.kerberos.KerberosServerAction.OperationType */ - private boolean regenerateAllKeytabs; + private KerberosServerAction.OperationType operationType; /** * A boolean value indicating whether to update service configurations (true) @@ -3897,6 +3870,14 @@ public class KerberosHelperImpl implements KerberosHelper { private boolean updateConfigurations; /** + * A boolean value indicating whether to include all hosts (true) when setting up + * agent-side tasks or to select only the hosts found to be relevant (false). + *

+ * This is useful if we do not know beforehand, which hosts need to be involved in the operation. + */ + private boolean forceAllHosts; + + /** * A boolean value indicating whether to include Ambari server identity (true) * or ignore it (false). */ @@ -3906,17 +3887,20 @@ public class KerberosHelperImpl implements KerberosHelper { * CreatePrincipalsAndKeytabsHandler constructor to set whether this instance should be used to * regenerate all keytabs or just the ones that have not been distributed * - * @param regenerateAllKeytabs A boolean value indicating whether to create keytabs for all - * principals (true or only the ones that are missing - * (false) - * @param updateConfigurations A boolean value indicating whether to update service configurations - * (true) or ignore any potential configuration changes - * (false) + * @param operationType The type of Kerberos operation being performed + * @param updateConfigurations A boolean value indicating whether to update service configurations + * (true) or ignore any potential configuration changes + * @param forceAllHosts A boolean value indicating whether to include all hosts (true) + * when setting up agent-side tasks or to select only the hosts found to be + * relevant (false) + * @param includeAmbariIdentity A boolean value indicating whether to include Ambari server + * identity (true) or ignore it (false) */ - public CreatePrincipalsAndKeytabsHandler(boolean regenerateAllKeytabs, boolean updateConfigurations, - boolean includeAmbariIdentity) { - this.regenerateAllKeytabs = regenerateAllKeytabs; + CreatePrincipalsAndKeytabsHandler(KerberosServerAction.OperationType operationType, boolean updateConfigurations, + boolean forceAllHosts, boolean includeAmbariIdentity) { + this.operationType = operationType; this.updateConfigurations = updateConfigurations; + this.forceAllHosts = forceAllHosts; this.includeAmbariIdentity = includeAmbariIdentity; } @@ -3947,6 +3931,7 @@ public class KerberosHelperImpl implements KerberosHelper { } + boolean processAmbariIdentity = includeAmbariIdentity; Map commandParameters = new HashMap<>(); commandParameters.put(KerberosServerAction.AUTHENTICATED_USER_NAME, ambariManagementController.getAuthName()); commandParameters.put(KerberosServerAction.DEFAULT_REALM, kerberosDetails.getDefaultRealm()); @@ -3955,22 +3940,29 @@ public class KerberosHelperImpl implements KerberosHelper { } if (serviceComponentFilter != null) { commandParameters.put(KerberosServerAction.SERVICE_COMPONENT_FILTER, StageUtils.getGson().toJson(serviceComponentFilter)); + + processAmbariIdentity = serviceComponentFilter.containsKey("AMBARI") && + ((serviceComponentFilter.get("AMBARI") == null) || serviceComponentFilter.get("AMBARI").contains("*") || serviceComponentFilter.get("AMBARI").contains("AMBARI_SERVER")); } if (hostFilter != null) { commandParameters.put(KerberosServerAction.HOST_FILTER, StageUtils.getGson().toJson(hostFilter)); + + processAmbariIdentity = hostFilter.contains("*") || hostFilter.contains(StageUtils.getHostName()); } if (identityFilter != null) { commandParameters.put(KerberosServerAction.IDENTITY_FILTER, StageUtils.getGson().toJson(identityFilter)); } - commandParameters.put(KerberosServerAction.REGENERATE_ALL, (regenerateAllKeytabs) ? "true" : "false"); - commandParameters.put(KerberosServerAction.INCLUDE_AMBARI_IDENTITY, (includeAmbariIdentity) ? "true" : "false"); + commandParameters.put(KerberosServerAction.OPERATION_TYPE, (operationType == null) ? KerberosServerAction.OperationType.DEFAULT.name() : operationType.name()); + commandParameters.put(KerberosServerAction.INCLUDE_AMBARI_IDENTITY, (processAmbariIdentity) ? "true" : "false"); if (updateConfigurations) { commandParameters.put(KerberosServerAction.UPDATE_CONFIGURATION_NOTE, "Updated Kerberos-related configurations"); commandParameters.put(KerberosServerAction.UPDATE_CONFIGURATIONS, "true"); } + List hostsToInclude = calculateHosts(cluster, serviceComponentHosts, hostsWithValidKerberosClient, forceAllHosts); + // ***************************************************************** // Create stage to create principals addPrepareKerberosIdentitiesStage(cluster, clusterHostInfoJson, hostParamsJson, event, @@ -3979,9 +3971,9 @@ public class KerberosHelperImpl implements KerberosHelper { if (kerberosDetails.manageIdentities()) { commandParameters.put(KerberosServerAction.KDC_TYPE, kerberosDetails.getKdcType().name()); - if (!regenerateAllKeytabs) { - addCheckMissingKeytabsStage(cluster, clusterHostInfoJson, hostParamsJson, event, - commandParameters, roleCommandOrder, requestStageContainer, serviceComponentHosts); + if (operationType != KerberosServerAction.OperationType.RECREATE_ALL) { + addCheckMissingKeytabsStage(cluster, clusterHostInfoJson, hostParamsJson, + commandParameters, roleCommandOrder, requestStageContainer, hostsToInclude); } // ***************************************************************** @@ -3996,15 +3988,15 @@ public class KerberosHelperImpl implements KerberosHelper { // ***************************************************************** // Create stage to distribute and configure keytab for Ambari server and configure JAAS - if (includeAmbariIdentity && kerberosDetails.createAmbariPrincipal()) { + if (processAmbariIdentity && kerberosDetails.createAmbariPrincipal()) { addConfigureAmbariIdentityStage(cluster, clusterHostInfoJson, hostParamsJson, event, commandParameters, roleCommandOrder, requestStageContainer); } // ***************************************************************** // Create stage to distribute keytabs - addDistributeKeytabFilesStage(cluster, serviceComponentHosts, clusterHostInfoJson, - hostParamsJson, commandParameters, roleCommandOrder, requestStageContainer, hostsWithValidKerberosClient); + addDistributeKeytabFilesStage(cluster, clusterHostInfoJson, hostParamsJson, commandParameters, + roleCommandOrder, requestStageContainer, hostsToInclude); } if (updateConfigurations) { @@ -4019,6 +4011,74 @@ public class KerberosHelperImpl implements KerberosHelper { } /** + * Filter out ServiceComponentHosts that are on on hosts in the specified set of host names. + *

+ * It is expected that the supplied collection is modifiable. It will be modified inplace. + * + * @param serviceComponentHosts a collection of ServiceComponentHost items to test + * @param hosts a set of host names indicating valid hosts + * @return a collection of filtered ServiceComponentHost items + */ + private Collection filterServiceComponentHostsForHosts(Collection serviceComponentHosts, + Set hosts) { + + if ((serviceComponentHosts != null) && (hosts != null)) { + Iterator iterator = serviceComponentHosts.iterator(); + while (iterator.hasNext()) { + ServiceComponentHost sch = iterator.next(); + + if (!hosts.contains(sch.getHostName())) { + iterator.remove(); + } + } + } + + return serviceComponentHosts; + } + + /** + * Calculate the hosts to include when issuing agent-side commands. + *

+ * If forcing all hosts, select only the healthy hosts in the cluster else select only the healthy + * hosts from the set of hosts specified in the collection of relevant {@link ServiceComponentHost}. + * + * @param cluster the cluster + * @param serviceComponentHosts a collction of {@link ServiceComponentHost}s that are + * relevant to the current operation + * @param hostsWithValidKerberosClient the collection of hosts know to have the Kerberos client + * component installed + * @param forceAllHosts true to process all hosts from the cluster rather than use + * the hosts parsed from the set of {@link ServiceComponentHost}s + * @return a filtered list of host names + * @throws AmbariException + */ + private List calculateHosts(Cluster cluster, List serviceComponentHosts, Set hostsWithValidKerberosClient, boolean forceAllHosts) throws AmbariException { + if(forceAllHosts) { + List hosts = new ArrayList<>(); + Collection clusterHosts = cluster.getHosts(); + if(!CollectionUtils.isEmpty(clusterHosts)) { + for(Host host: clusterHosts) { + if(host.getState() == HostState.HEALTHY) { + hosts.add(host.getHostName()); + } + } + } + + return hosts; + } + else { + Collection filteredComponents = filterServiceComponentHostsForHosts( + new ArrayList<>(serviceComponentHosts), hostsWithValidKerberosClient); + + if (filteredComponents.isEmpty()) { + return Collections.emptyList(); + } else { + return createUniqueHostList(filteredComponents, Collections.singleton(HostState.HEALTHY)); + } + } + } + + /** * DeletePrincipalsAndKeytabsHandler is an implementation of the Handler interface used to delete * principals and keytabs throughout the cluster. *

http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java index 66bf7b3..cd23e83 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/RemovableIdentities.java @@ -133,7 +133,7 @@ public class RemovableIdentities { * Remove all identities which are not used by other services or components */ public void remove(KerberosHelper kerberosHelper) throws AmbariException, KerberosOperationException { - Set identitiesToRemove = skipUsed().stream().map(KerberosIdentityDescriptor::getName).collect(toSet()); + Set identitiesToRemove = skipUsed().stream().map(KerberosIdentityDescriptor::getPath).collect(toSet()); if (!identitiesToRemove.isEmpty()) { kerberosHelper.deleteIdentities(cluster, components, identitiesToRemove); } http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java index 4396a2b..355f515 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreateKeytabFilesServerAction.java @@ -217,7 +217,7 @@ public class CreateKeytabFilesServerAction extends KerberosServerAction { return commandReport; } - boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL)); + boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL; boolean onlyKeytabWrite = "true".equalsIgnoreCase(identityRecord.get(KerberosIdentityDataFileReader.ONLY_KEYTAB_WRITE)); boolean grabKeytabFromCache = regenerateKeytabs && onlyKeytabWrite; // if grabKeytabFromCache=true we will try to get keytab from cache and send to agent, it will be true for http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java index 069c821..1c0853b9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java @@ -128,7 +128,7 @@ public class CreatePrincipalsServerAction extends KerberosServerAction { seenPrincipals.add(evaluatedPrincipal); boolean processPrincipal; - boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL)); + boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL; if (regenerateKeytabs) { // do not process cached identities that can be passed as is(headless identities) http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java index c86ffa3..1b0f4fb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/KerberosServerAction.java @@ -36,6 +36,7 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,31 +109,32 @@ public abstract class KerberosServerAction extends AbstractServerAction { */ public static final String DATA_DIRECTORY_PREFIX = ".ambari_"; - /* + /** * Kerberos action shared data entry name for the principal-to-password map */ private static final String PRINCIPAL_PASSWORD_MAP = "principal_password_map"; - /* + /** * Kerberos action shared data entry name for the principal-to-key_number map */ private static final String PRINCIPAL_KEY_NUMBER_MAP = "principal_key_number_map"; - /* - * Key used in kerberosCommandParams in ExecutionCommand for base64 encoded keytab content - */ + /** + * Key used in kerberosCommandParams in ExecutionCommand for base64 encoded keytab content + */ public static final String KEYTAB_CONTENT_BASE64 = "keytab_content_base64"; - /* - * Key used in kerberosCommandParams in ExecutionCommand to indicate whether to generate key keytabs - * for all principals ("true") or only those that are missing ("false") - */ - public static final String REGENERATE_ALL = "regenerate_all"; + /** + * Key used in kerberosCommandParams in ExecutionCommand to indicate why type of creation operation to perform. + * + * @see OperationType + */ + public static final String OPERATION_TYPE = "operation_type"; - /* - * Key used in kerberosCommandParams in ExecutionCommand to indicate whether to include Ambari server indetity - * ("true") or ignore it ("false") - */ + /** + * Key used in kerberosCommandParams in ExecutionCommand to indicate whether to include Ambari server indetity + * ("true") or ignore it ("false") + */ public static final String INCLUDE_AMBARI_IDENTITY = "include_ambari_identity"; /** @@ -219,6 +221,22 @@ public abstract class KerberosServerAction extends AbstractServerAction { } /** + * Given a (command parameter) Map, attempts to safely retrieve the "operation_type" property. + * + * @param commandParameters a Map containing the dictionary of data to interrogate + * @return an OperationType + */ + protected static OperationType getOperationType(Map commandParameters) { + String value = getCommandParameterValue(commandParameters, OPERATION_TYPE); + if(StringUtils.isEmpty(value)) { + return OperationType.DEFAULT; + } + else { + return OperationType.valueOf(value.toUpperCase()); + } + } + + /** * Sets the shared principal-to-password Map used to store principals and generated password for * use within the current request context. * @@ -569,4 +587,29 @@ public abstract class KerberosServerAction extends AbstractServerAction { } } } + + /** + * A Kerberos operation type + *

    + *
  • RECREATE_ALL - regenerate keytabs for all principals
    • + *
  • CREATE_MISSING - generate keytabs for only those that are missing
    • + *
  • DEFAULT - generate needed keytabs for new components
    • + *
+ */ + public enum OperationType { + /** + * Regenerate keytabs for all principals + */ + RECREATE_ALL, + + /** + * Generate keytabs for only those that are missing + */ + CREATE_MISSING, + + /** + * Generate needed keytabs for new components + */ + DEFAULT + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java index f56e946..e1f8419 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java @@ -83,8 +83,7 @@ public class PrepareDisableKerberosServerAction extends AbstractPrepareKerberosS List schToProcess = kerberosHelper.getServiceComponentHostsToProcess(cluster, kerberosDescriptor, getServiceComponentFilter(), - null, identityFilter, - sch -> true); + null); Map> kerberosConfigurations = new HashMap<>(); Map commandParameters = getCommandParameters(); http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java index 3ec84fa..335451f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java @@ -92,8 +92,11 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities } } + KerberosHelper kerberosHelper = getKerberosHelper(); + Map> serviceComponentFilter = getServiceComponentFilter(); + Collection hostFilter = getHostFilter(); Collection identityFilter = getIdentityFilter(); - List schToProcess = getServiceComponentHostsToProcess(cluster, kerberosDescriptor, identityFilter); + List schToProcess = kerberosHelper.getServiceComponentHostsToProcess(cluster, kerberosDescriptor, serviceComponentFilter, hostFilter); String dataDirectory = getCommandParameterValue(commandParameters, DATA_DIRECTORY); Map> kerberosConfigurations = new HashMap<>(); @@ -107,7 +110,6 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities actionLog.writeStdOut(String.format("Processing %d components", schCount)); } - KerberosHelper kerberosHelper = getKerberosHelper(); Map> propertiesToRemove = new HashMap<>(); Map> propertiesToIgnore = new HashMap<>(); Set services = cluster.getServices().keySet(); http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java index 49828cb..038d1b5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java @@ -32,8 +32,12 @@ import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.controller.KerberosHelper; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor; import org.apache.ambari.server.state.kerberos.KerberosDescriptor; +import org.apache.ambari.server.state.kerberos.KerberosIdentityDescriptor; +import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,11 +71,22 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber throw new AmbariException("Missing cluster object"); } + KerberosHelper kerberosHelper = getKerberosHelper(); + KerberosDescriptor kerberosDescriptor = getKerberosDescriptor(cluster, false); + Map commandParameters = getCommandParameters(); + OperationType operationType = getOperationType(getCommandParameters()); + + Map> serviceComponentFilter = getServiceComponentFilter(); + Collection hostFilter = getHostFilter(); Collection identityFilter = getIdentityFilter(); - List schToProcess = getServiceComponentHostsToProcess(cluster, kerberosDescriptor, identityFilter); + // If the operationType is default, use the getServiceComponentHostsToProcess method to determine + // which ServiceComponentHosts to process based on the filters. However if we are regenerating + // keytabs for a specific set of components, build the identity filter below so we can + // customized what needs to be done. + List schToProcess = kerberosHelper.getServiceComponentHostsToProcess(cluster, kerberosDescriptor, + (operationType == OperationType.DEFAULT) ? serviceComponentFilter : null, hostFilter); - Map commandParameters = getCommandParameters(); String dataDirectory = getCommandParameterValue(commandParameters, DATA_DIRECTORY); Map> kerberosConfigurations = new HashMap<>(); @@ -84,18 +99,32 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber actionLog.writeStdOut(String.format("Processing %d components", schCount)); } - KerberosHelper kerberosHelper = getKerberosHelper(); Set services = cluster.getServices().keySet(); Map> propertiesToRemove = new HashMap<>(); Map> propertiesToIgnore = new HashMap<>(); boolean includeAmbariIdentity = "true".equalsIgnoreCase(getCommandParameterValue(commandParameters, KerberosServerAction.INCLUDE_AMBARI_IDENTITY)); + // If we are including the Ambari identity; then ensure that if a host filter is set, do not the Ambari service identity. + includeAmbariIdentity &= (hostFilter == null); + + if (serviceComponentFilter != null) { + // If we are including the Ambari identity; then ensure that if a service/component filter is set, + // it contains the AMBARI/AMBARI_SERVER component; else do not include the Ambari service identity. + includeAmbariIdentity &= (serviceComponentFilter.get("AMBARI") != null) && serviceComponentFilter.get("AMBARI").contains("AMBARI_SERVER"); + + if((operationType != OperationType.DEFAULT)) { + // Update the identity filter, if necessary + identityFilter = updateIdentityFilter(kerberosDescriptor, identityFilter, serviceComponentFilter); + } + } + // Calculate the current host-specific configurations. These will be used to replace // variables within the Kerberos descriptor data Map> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptor, false, false); processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, - configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore, !CollectionUtils.isEmpty(getHostFilter())); + configurations, kerberosConfigurations, includeAmbariIdentity, propertiesToIgnore, + hostFilter != null); kerberosHelper.applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, propertiesToIgnore, propertiesToRemove, true); @@ -119,35 +148,6 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber } /** - * Calls {@link KerberosHelper#getServiceComponentHostsToProcess(Cluster, KerberosDescriptor, Map, Collection, Collection, KerberosHelper.Command)} - * with no filter on ServiceComponentHosts - *

- * The shouldProcessCommand implementation passed to KerberosHelper#getServiceComponentHostsToProcess - * always returns true, indicating to process all ServiceComponentHosts. - * - * @param cluster the cluster - * @param kerberosDescriptor the current Kerberos descriptor - * @param identityFilter a list of identities to include, or all if null @return the list of ServiceComponentHosts to process - * @throws AmbariException - * @see KerberosHelper#getServiceComponentHostsToProcess(Cluster, KerberosDescriptor, Map, Collection, Collection, KerberosHelper.Command) - */ - protected List getServiceComponentHostsToProcess(Cluster cluster, - KerberosDescriptor kerberosDescriptor, - Collection identityFilter) - throws AmbariException { - return getKerberosHelper().getServiceComponentHostsToProcess(cluster, - kerberosDescriptor, - getServiceComponentFilter(), - getHostFilter(), identityFilter, - new KerberosHelper.Command() { - @Override - public Boolean invoke(ServiceComponentHost sch) throws AmbariException { - return true; - } - }); - } - - /** * Calls {@link KerberosHelper#getKerberosDescriptor(Cluster, boolean)} * * @param cluster cluster instance @@ -200,5 +200,81 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber calculatedConfiguration, kerberosConfigurations, includePreconfiguredData); } } + + /** + * Iterate through the identities in the Kerberos descriptor to find the relevant identities to + * add to the identity filter. + *

+ * The set of identities to include in the filter are determined by whether they are explicit + * identities set in a component or service in the supplied service/component filter. + * + * @param kerberosDescriptor the Kerberos descriptor + * @param identityFilter the existing identity filter + * @param serviceComponentFilter the service/component filter + * @return a new collection of paths (including any existing paths) to act as the updated identity filter + */ + private Collection updateIdentityFilter(KerberosDescriptor kerberosDescriptor, + Collection identityFilter, + Map> serviceComponentFilter) { + + Set updatedFilter = (identityFilter == null) ? new HashSet<>() : new HashSet<>(identityFilter); + + Map serviceDescriptors = kerberosDescriptor.getServices(); + + if (serviceDescriptors != null) { + for (KerberosServiceDescriptor serviceDescriptor : serviceDescriptors.values()) { + String serviceName = serviceDescriptor.getName(); + + if (serviceComponentFilter.containsKey("*") || serviceComponentFilter.containsKey(serviceName)) { + Collection componentFilter = serviceComponentFilter.get(serviceName); + boolean anyComponent = ((componentFilter == null) || componentFilter.contains("*")); + + // Only include the service-wide identities if the component filter is null contains "*", which indicates + // that all component for the given service are to be processed. + if (anyComponent) { + addIdentitiesToFilter(serviceDescriptor.getIdentities(), updatedFilter, true); + } + + Map componentDescriptors = serviceDescriptor.getComponents(); + if (componentDescriptors != null) { + for (KerberosComponentDescriptor componentDescriptor : componentDescriptors.values()) { + String componentName = componentDescriptor.getName(); + if (anyComponent || (componentFilter.contains(componentName))) { + addIdentitiesToFilter(componentDescriptor.getIdentities(), updatedFilter, true); + } + } + } + } + } + } + + return updatedFilter; + } + + /** + * Add the path of each identity in the collection of identities to the supplied identity filter + * if that identity is not a reference to another identity or if references are allowed. + * @param identityDescriptors the collection of identity descriptors to process + * @param identityFilter the identity filter to modify + * @param skipReferences + */ + private void addIdentitiesToFilter(List identityDescriptors, + Collection identityFilter, boolean skipReferences) { + if (!CollectionUtils.isEmpty(identityDescriptors)) { + for (KerberosIdentityDescriptor identityDescriptor : identityDescriptors) { + if (!skipReferences || !identityDescriptor.isReference()) { + String identityPath = identityDescriptor.getPath(); + + if (!StringUtils.isEmpty(identityPath)) { + identityFilter.add(identityPath); + + // Find and add the references TO this identity to ensure the new/updated keytab file is + // sent to the appropriate host(s) + addIdentitiesToFilter(identityDescriptor.findReferences(), identityFilter, false); + } + } + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java index b496942..3a1eb4a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptor.java @@ -259,6 +259,31 @@ public abstract class AbstractKerberosDescriptor { } /** + * Calculate the path to this identity descriptor for logging purposes. + * Examples: + *

    + *
  • /
    • + *
  • /SERVICE
    • + *
  • /SERVICE/COMPONENT
    • + *
  • /SERVICE/COMPONENT/identity_name
    • + *
+ * + * @return a path + */ + public String getPath() { + // + StringBuilder path = new StringBuilder(); + AbstractKerberosDescriptor current = this; + while (current != null && (current.getName() != null)) { + path.insert(0, current.getName()); + path.insert(0, '/'); + current = current.getParent(); + } + + return path.toString(); + } + + /** * An enumeration of the different Kerberos (sub)descriptors for internal use. */ public enum Type { http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java index 9ddb941..73550f4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/AbstractKerberosDescriptorContainer.java @@ -862,22 +862,9 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber referencedIdentity = getReferencedIdentityDescriptor(identity.getName()); if(referencedIdentity != null) { - // Calculate the path to this identity descriptor for logging purposes. - // Examples: - // / - // /SERVICE - // /SERVICE/COMPONENT - StringBuilder path = new StringBuilder(); - AbstractKerberosDescriptor parent = identity.getParent(); - while(parent != null && (parent.getName() != null)) { - path.insert(0, parent.getName()); - path.insert(0, '/'); - parent = parent.getParent(); - } - // Log this since it is deprecated... LOG.warn("Referenced identities should be declared using the identity's \"reference\" attribute, not the identity's \"name\" attribute." + - " This is a deprecated feature. Problems may occur in the future unless this is corrected: {}:{}", path, identity.getName()); + " This is a deprecated feature. Problems may occur in the future unless this is corrected: {}:{}", identity.getPath(), identity.getName()); } } } catch (AmbariException e) { @@ -896,6 +883,9 @@ public abstract class AbstractKerberosDescriptorContainer extends AbstractKerber } else { dereferencedIdentity = new KerberosIdentityDescriptor(identity.toMap()); } + + // Force the path for this identity descriptor to be the same as the original identity descriptor's. + dereferencedIdentity.setPath(identity.getPath()); } return dereferencedIdentity; http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java index ef45343..200a069 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/kerberos/KerberosIdentityDescriptor.java @@ -17,10 +17,15 @@ */ package org.apache.ambari.server.state.kerberos; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.Map; import org.apache.ambari.server.collections.Predicate; import org.apache.ambari.server.collections.PredicateUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import com.google.common.base.Optional; @@ -94,6 +99,8 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor { */ private Predicate when = null; + private String path = null; + /** * Creates a new KerberosIdentityDescriptor * @@ -157,6 +164,47 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor { } /** + * Gets the absolute path to the referenced Kerberos identity definition + * + * @return the path to the referenced Kerberos identity definition or null if not set + */ + public String getReferenceAbsolutePath() { + String absolutePath; + if(StringUtils.isEmpty(reference)) { + absolutePath = getName(); + } + else { + absolutePath = reference; + } + + if(!StringUtils.isEmpty(absolutePath) && !absolutePath.startsWith("/")) { + String path = getPath(); + if(path == null) { + path = ""; + } + + if(absolutePath.startsWith("..")) { + AbstractKerberosDescriptor parent = getParent(); + if(parent != null) { + parent = parent.getParent(); + + if(parent != null) { + absolutePath = absolutePath.replace("..", parent.getPath()); + } + } + } + else if(absolutePath.startsWith(".")) { + AbstractKerberosDescriptor parent = getParent(); + if (parent != null) { + absolutePath = absolutePath.replace(".", parent.getPath()); + } + } + } + + return absolutePath; + } + + /** * Sets the path to the referenced Kerberos identity definition * * @param reference the path to the referenced Kerberos identity definition or null @@ -356,6 +404,59 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor { } } + /** + * Determines whether this {@link KerberosIdentityDescriptor} indicates it is a refrence to some + * other {@link KerberosIdentityDescriptor}. + *

+ * A KerberosIdentityDescriptor is a reference if it's reference attibute is set + * or if (for backwards compatibility), its name indicates a path. For exmaple: + *

    + *
  • SERVICE/COMPONENT/identitiy_name
    • + *
  • /identity_name
    • + *
  • ./identity_name
    • + *
+ * + * @return true if this {@link KerberosIdentityDescriptor} indicates a reference; otherwise false + */ + public boolean isReference() { + String name = getName(); + return !StringUtils.isEmpty(reference) || + (!StringUtils.isEmpty(name) && (name.startsWith("/") || name.startsWith("./"))); + } + + /** + * Calculate the path to this identity descriptor for logging purposes. + * Examples: + * / + * /SERVICE + * /SERVICE/COMPONENT + * /SERVICE/COMPONENT/identity_name + *

+ * This implementation calculates and caches the path if the path has not been previously set. + * + * @return a path + */ + @Override + public String getPath() { + if (path == null) { + path = super.getPath(); + } + + return path; + } + + /** + * Explicitly set the path to this {@link KerberosIdentityDescriptor}. + *

+ * This is useful when creating detached identity descriptors while dereferencing identity references + * so that the path information is not lost. + * + * @param path a path + */ + void setPath(String path) { + this.path = path; + } + @Override public int hashCode() { return super.hashCode() + @@ -406,4 +507,63 @@ public class KerberosIdentityDescriptor extends AbstractKerberosDescriptor { return false; } } + + /** + * Find all of the {@link KerberosIdentityDescriptor}s that reference this {@link KerberosIdentityDescriptor} + * + * @return a list of {@link KerberosIdentityDescriptor}s + */ + public List findReferences() { + AbstractKerberosDescriptor root = getRoot(); + if(root instanceof AbstractKerberosDescriptorContainer) { + return findIdentityReferences((AbstractKerberosDescriptorContainer)root, getPath()); + } + else { + return null; + } + } + + /** + * Given a root, recursively traverse the tree of {@link AbstractKerberosDescriptorContainer}s looking for + * {@link KerberosIdentityDescriptor}s that declare the given path as the referenced Kerberos identity. + * + * @param root the starting point + * @param path the path to the referenced {@link KerberosIdentityDescriptor} in the {@link KerberosDescriptor} + * @return a list of {@link KerberosIdentityDescriptor}s + */ + private List findIdentityReferences(AbstractKerberosDescriptorContainer root, String path) { + if (root == null) { + return null; + } + + List references = new ArrayList<>(); + + // Process the KerberosIdentityDescriptors found in this node. + List identityDescriptors = root.getIdentities(); + if (identityDescriptors != null) { + for (KerberosIdentityDescriptor identityDescriptor : identityDescriptors) { + if (identityDescriptor.isReference()) { + String reference = identityDescriptor.getReferenceAbsolutePath(); + + if (!StringUtils.isEmpty(reference) && path.equals(reference)) { + references.add(identityDescriptor); + } + } + } + } + + // Process the children of the node + Collection children = root.getChildContainers(); + if(!CollectionUtils.isEmpty(children)) { + for (AbstractKerberosDescriptorContainer child : children) { + Collection childReferences = findIdentityReferences(child, path); + if (!CollectionUtils.isEmpty(childReferences)) { + // If references were found in the current child, add them to this node's list of references. + references.addAll(childReferences); + } + } + } + + return references; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java index 60d7fd9..7ed52d2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java @@ -1441,11 +1441,6 @@ public class KerberosHelperTest extends EasyMockSupport { .andReturn(Collections.singletonList(schKerberosClient)) .once(); - final Clusters clusters = injector.getInstance(Clusters.class); - expect(clusters.getHost("host1")) - .andReturn(host) - .once(); - final AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); expect(ambariManagementController.findConfigurationTagsWithOverrides(cluster, null)) .andReturn(Collections.emptyMap()) http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java index 663934f..2518da9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/KerberosIdentityCleanerTest.java @@ -77,7 +77,7 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport { @Test public void removesAllKerberosIdentitesOfComponentAfterComponentWasUninstalled() throws Exception { installComponent(OOZIE, OOZIE_SERVER, HOST); - kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, OOZIE, OOZIE_SERVER)), newHashSet("oozie_server1", "oozie_server2")); + kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, OOZIE, OOZIE_SERVER)), newHashSet("/OOZIE/OOZIE_SERVER/oozie_server1", "/OOZIE/OOZIE_SERVER/oozie_server2")); expectLastCall().once(); replayAll(); uninstallComponent(OOZIE, OOZIE_SERVER, HOST); @@ -95,7 +95,7 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport { public void skipsRemovingIdentityThatIsSharedByPrincipalName() throws Exception { installComponent(OOZIE, OOZIE_SERVER, HOST); installComponent(OOZIE_2, OOZIE_SERVER_2, HOST); - kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, OOZIE, OOZIE_SERVER)), newHashSet("oozie_server1")); + kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, OOZIE, OOZIE_SERVER)), newHashSet("/OOZIE/OOZIE_SERVER/oozie_server1")); expectLastCall().once(); replayAll(); uninstallComponent(OOZIE, OOZIE_SERVER, HOST); @@ -106,7 +106,7 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport { public void skipsRemovingIdentityThatIsSharedByKeyTabFilePath() throws Exception { installComponent(YARN, RESOURCE_MANAGER, HOST); installComponent(YARN_2, RESOURCE_MANAGER_2, HOST); - kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, YARN, RESOURCE_MANAGER)), newHashSet("rm_unique")); + kerberosHelper.deleteIdentities(cluster, singletonList(new Component(HOST, YARN, RESOURCE_MANAGER)), newHashSet("/YARN/RESOURCE_MANAGER/rm_unique")); expectLastCall().once(); replayAll(); uninstallComponent(YARN, RESOURCE_MANAGER, HOST); @@ -133,7 +133,7 @@ public class KerberosIdentityCleanerTest extends EasyMockSupport { @Test public void removesServiceIdentitiesSkipComponentIdentitiesAfterServiceWasUninstalled() throws Exception { installComponent(OOZIE, OOZIE_SERVER, HOST); - kerberosHelper.deleteIdentities(cluster, hdfsComponents(), newHashSet("hdfs-service")); + kerberosHelper.deleteIdentities(cluster, hdfsComponents(), newHashSet("/HDFS/hdfs-service")); expectLastCall().once(); replayAll(); uninstallService(HDFS, hdfsComponents()); http://git-wip-us.apache.org/repos/asf/ambari/blob/5af1e539/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java index d6bef02..079096d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/kerberos/KerberosDescriptorTest.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -79,6 +80,9 @@ public class KerberosDescriptorTest { " {" + " \"name\": \"service1_spnego\"," + " \"reference\": \"/spnego\"" + + " }," + + " {" + + " \"name\": \"service1_identity\"" + " }" + " ]," + " \"name\": \"SERVICE1\"" + @@ -87,6 +91,39 @@ public class KerberosDescriptorTest { " \"identities\": [" + " {" + " \"name\": \"/spnego\"" + + " }," + + " {" + + " \"name\": \"service2_identity\"" + + " }" + + " ]," + + " \"components\": [" + + " {" + + " \"identities\": [" + + " {" + + " \"name\": \"component1_identity\"" + + " }," + + " {" + + " \"name\": \"service2_component1_service1_identity\"," + + " \"reference\": \"/SERVICE1/service1_identity\"" + + " }," + + " {" + + " \"name\": \"service2_component1_component1_identity\"," + + " \"reference\": \"./component1_identity\"" + + " }," + + " {" + + " \"name\": \"service2_component1_service2_identity\"," + + " \"reference\": \"../service2_identity\"" + + " }" + + " ]," + + " \"name\": \"COMPONENT21\"" + + " }," + + " {" + + " \"identities\": [" + + " {" + + " \"name\": \"component2_identity\"" + + " }" + + " ]," + + " \"name\": \"COMPONENT22\"" + " }" + " ]," + " \"name\": \"SERVICE2\"" + @@ -547,15 +584,118 @@ public class KerberosDescriptorTest { // Reference is determined using the "reference" attribute serviceDescriptor = kerberosDescriptor.getService("SERVICE1"); identities = serviceDescriptor.getIdentities(true, null); - Assert.assertEquals(1, identities.size()); - Assert.assertEquals("service1_spnego", identities.get(0).getName()); - Assert.assertEquals("/spnego", identities.get(0).getReference()); + Assert.assertEquals(2, identities.size()); + for (KerberosIdentityDescriptor identity : identities) { + if (identity.isReference()) { + Assert.assertEquals("service1_spnego", identity.getName()); + Assert.assertEquals("/spnego", identity.getReference()); + } else { + Assert.assertEquals("service1_identity", identity.getName()); + Assert.assertNull(identity.getReference()); + } + } + + Assert.assertEquals("service1_identity", identities.get(1).getName()); + Assert.assertNull(identities.get(1).getReference()); // Reference is determined using the "name" attribute serviceDescriptor = kerberosDescriptor.getService("SERVICE2"); identities = serviceDescriptor.getIdentities(true, null); + Assert.assertEquals(2, identities.size()); + for (KerberosIdentityDescriptor identity : identities) { + if (identity.isReference()) { + Assert.assertEquals("/spnego", identity.getName()); + Assert.assertNull(identity.getReference()); + } else { + Assert.assertEquals("service2_identity", identity.getName()); + Assert.assertNull(identity.getReference()); + } + } + } + + @Test + public void testGetPath() throws Exception { + KerberosDescriptor kerberosDescriptor; + KerberosServiceDescriptor serviceDescriptor; + List identities; + + kerberosDescriptor = KERBEROS_DESCRIPTOR_FACTORY.createInstance(JSON_VALUE); + + serviceDescriptor = kerberosDescriptor.getService("SERVICE_NAME"); + identities = serviceDescriptor.getIdentities(false, null); + Assert.assertEquals(1, identities.size()); + Assert.assertEquals("/SERVICE_NAME/identity_1", identities.get(0).getPath()); + + KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent("COMPONENT_NAME"); + identities = componentDescriptor.getIdentities(false, null); Assert.assertEquals(1, identities.size()); - Assert.assertEquals("/spnego", identities.get(0).getName()); - Assert.assertNull(identities.get(0).getReference()); + Assert.assertEquals("/SERVICE_NAME/COMPONENT_NAME/identity_1", identities.get(0).getPath()); + + + kerberosDescriptor = KERBEROS_DESCRIPTOR_FACTORY.createInstance(JSON_VALUE_IDENTITY_REFERENCES); + + serviceDescriptor = kerberosDescriptor.getService("SERVICE1"); + identities = serviceDescriptor.getIdentities(true, null); + Assert.assertEquals(2, identities.size()); + Assert.assertEquals("/SERVICE1/service1_spnego", identities.get(0).getPath()); + Assert.assertEquals("/SERVICE1/service1_identity", identities.get(1).getPath()); + } + + @Test + public void testGetReferences() throws Exception { + KerberosDescriptor kerberosDescriptor = KERBEROS_DESCRIPTOR_FACTORY.createInstance(JSON_VALUE_IDENTITY_REFERENCES); + KerberosIdentityDescriptor identity; + List references; + Set paths; + + // Find all references to /spnego + identity = kerberosDescriptor.getIdentity("spnego"); + references = identity.findReferences(); + + Assert.assertNotNull(references); + Assert.assertEquals(2, references.size()); + + paths = collectPaths(references); + Assert.assertTrue(paths.contains("/SERVICE1/service1_spnego")); + Assert.assertTrue(paths.contains("/SERVICE2//spnego")); + + // Find all references to /SERVICE1/service1_identity + identity = kerberosDescriptor.getService("SERVICE1").getIdentity("service1_identity"); + references = identity.findReferences(); + + Assert.assertNotNull(references); + Assert.assertEquals(1, references.size()); + + paths = collectPaths(references); + Assert.assertTrue(paths.contains("/SERVICE2/COMPONENT21/service2_component1_service1_identity")); + + // Find all references to /SERVICE2/COMPONENT21/component1_identity (testing ./) + identity = kerberosDescriptor.getService("SERVICE2").getComponent("COMPONENT21").getIdentity("component1_identity"); + references = identity.findReferences(); + + Assert.assertNotNull(references); + Assert.assertEquals(1, references.size()); + + paths = collectPaths(references); + Assert.assertTrue(paths.contains("/SERVICE2/COMPONENT21/service2_component1_component1_identity")); + + // Find all references to /SERVICE2/component2_identity (testing ../) + identity = kerberosDescriptor.getService("SERVICE2").getIdentity("service2_identity"); + references = identity.findReferences(); + + Assert.assertNotNull(references); + Assert.assertEquals(1, references.size()); + + paths = collectPaths(references); + Assert.assertTrue(paths.contains("/SERVICE2/COMPONENT21/service2_component1_service2_identity")); + } + + private Set collectPaths(List identityDescriptors) { + Set paths = new HashSet<>(); + for (KerberosIdentityDescriptor identityDescriptor : identityDescriptors) { + paths.add(identityDescriptor.getPath()); + } + return paths; } + } \ No newline at end of file