Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6994E200BCA for ; Mon, 21 Nov 2016 22:11:51 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 6819D160AEF; Mon, 21 Nov 2016 21:11:51 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3C155160AF9 for ; Mon, 21 Nov 2016 22:11:50 +0100 (CET) Received: (qmail 58895 invoked by uid 500); 21 Nov 2016 21:11:49 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 58886 invoked by uid 99); 21 Nov 2016 21:11:49 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Nov 2016 21:11:49 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4D040E38F9; Mon, 21 Nov 2016 21:11:49 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Message-Id: <9e14931f58744ee3887dcf3552903f6c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-18937. Allow generic Service Upgrade Packs for all targets (ncole) Date: Mon, 21 Nov 2016 21:11:49 +0000 (UTC) archived-at: Mon, 21 Nov 2016 21:11:51 -0000 Repository: ambari Updated Branches: refs/heads/trunk ef91365b7 -> 7d07b71de AMBARI-18937. Allow generic Service Upgrade Packs for all targets (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7d07b71d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7d07b71d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7d07b71d Branch: refs/heads/trunk Commit: 7d07b71de7019efaaa04fe8c09548cdc604cea60 Parents: ef91365 Author: Nate Cole Authored: Mon Nov 21 15:24:59 2016 -0500 Committer: Nate Cole Committed: Mon Nov 21 16:11:21 2016 -0500 ---------------------------------------------------------------------- .../apache/ambari/server/stack/StackModule.java | 101 +++++++++++++++++-- .../ambari/server/state/stack/UpgradePack.java | 12 ++- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 4 +- .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 4 +- .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 4 +- .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 4 +- .../src/main/resources/upgrade-pack.xsd | 10 +- .../server/state/stack/UpgradePackTest.java | 22 ++++ .../GANGLIA/upgrades/HDP/rolling-upgrade.xml | 27 +++++ 9 files changed, 168 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index d9eaf27..ce606de 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -46,16 +46,19 @@ import org.apache.ambari.server.state.stack.ServiceMetainfoXml; import org.apache.ambari.server.state.stack.StackMetainfoXml; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.Grouping; +import org.apache.ambari.server.state.stack.upgrade.UpgradeType; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Function; -import com.google.common.collect.ListMultimap; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimaps; + /** * Stack module which provides all functionality related to parsing and fully * resolving stacks from the stack definition. @@ -834,6 +837,7 @@ public class StackModule extends BaseModule implements V if (configPack == null) { return; } + for (ServiceModule module : serviceModules.values()) { File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder(); if (upgradesFolder != null) { @@ -869,12 +873,17 @@ public class StackModule extends BaseModule implements V private UpgradePack getServiceUpgradePack(UpgradePack pack, File upgradesFolder) throws AmbariException { File stackFolder = new File(upgradesFolder, stackInfo.getName()); File versionFolder = new File(stackFolder, stackInfo.getVersion()); + // !!! relies on the service upgrade pack filename being named the exact same File servicePackFile = new File(versionFolder, pack.getName() + ".xml"); + LOG.info("Service folder: " + servicePackFile.getAbsolutePath()); - if (!servicePackFile.exists()) { - return null; + if (servicePackFile.exists()) { + return parseServiceUpgradePack(pack, servicePackFile); + } else { + UpgradePack child = findServiceUpgradePack(pack, stackFolder); + + return null == child ? null : parseServiceUpgradePack(pack, child); } - return parseServiceUpgradePack(pack, servicePackFile); } /** @@ -882,6 +891,7 @@ public class StackModule extends BaseModule implements V * for any service which specifies its own upgrade. */ private void mergeUpgradePack(UpgradePack pack, List servicePacks) throws AmbariException { + List originalGroups = pack.getAllGroups(); Map> allGroupMap = new HashMap<>(); for (Grouping group : originalGroups) { @@ -889,8 +899,21 @@ public class StackModule extends BaseModule implements V list.add(group); allGroupMap.put(group.name, list); } + for (UpgradePack servicePack : servicePacks) { for (Grouping group : servicePack.getAllGroups()) { + + /* + !!! special case where the service pack is targeted for any version. When + a service UP targets to run after another group, check to make sure that the + base UP contains the group. + */ + if (servicePack.isAllTarget() && !allGroupMap.keySet().contains(group.addAfterGroup)) { + LOG.warn("Service Upgrade Pack specified after-group of {}, but that is not found in {}", + group.addAfterGroup, StringUtils.join(allGroupMap.keySet(), ',')); + continue; + } + if (allGroupMap.containsKey(group.name)) { List list = allGroupMap.get(group.name); Grouping first = list.get(0); @@ -906,8 +929,7 @@ public class StackModule extends BaseModule implements V else { list.add(group); } - } - else { + } else { List list = new ArrayList<>(); list.add(group); allGroupMap.put(group.name, list); @@ -934,15 +956,17 @@ public class StackModule extends BaseModule implements V */ private void orderGroups(List groups, Map mergedGroupMap) throws AmbariException { Map> skippedGroups = new HashMap<>(); + for (Map.Entry entry : mergedGroupMap.entrySet()) { - String key = entry.getKey(); Grouping group = entry.getValue(); + if (!groups.contains(group)) { boolean added = addGrouping(groups, group); if (added) { addSkippedGroup(groups, skippedGroups, group); } else { List tmp = null; + // store the group until later if (skippedGroups.containsKey(group.addAfterGroup)) { tmp = skippedGroups.get(group.addAfterGroup); @@ -954,6 +978,7 @@ public class StackModule extends BaseModule implements V } } } + if (!skippedGroups.isEmpty()) { throw new AmbariException("Missing groups: " + skippedGroups.keySet()); } @@ -999,6 +1024,50 @@ public class StackModule extends BaseModule implements V } /** + * Finds an upgrade pack that: + *
    + *
  • Is found in the $SERVICENAME/upgrades/$STACKNAME folder
    • + *
  • Matches the same {@link UpgradeType#getType()}as the {@code base} upgrade pack
    • + *
  • Has the {@link UpgradePack#getTarget()} value equals to "*"
    • + *
  • Has the {@link UpgradePack#getTargetStack()} value equals to "*"
    • + *
+ * This method will not attempt to resolve the "most correct" upgrade pack. For this + * feature to work, there should be only one upgrade pack per type. If more specificity + * is required, then follow the convention of $SERVICENAME/upgrades/$STACKNAME/$STACKVERSION/$BASE_FILE_NAME.xml + * + * @param base the base upgrade pack for a stack + * @param upgradeStackDirectory service directory that contains stack upgrade files. + * @return an upgrade pack that matches {@code base} + */ + private UpgradePack findServiceUpgradePack(UpgradePack base, File upgradeStackDirectory) { + if (!upgradeStackDirectory.exists() || !upgradeStackDirectory.isDirectory()) { + return null; + } + + File[] upgradeFiles = upgradeStackDirectory.listFiles(StackDirectory.XML_FILENAME_FILTER); + if (0 == upgradeFiles.length) { + return null; + } + + for (File f : upgradeFiles) { + try { + UpgradePack upgradePack = unmarshaller.unmarshal(UpgradePack.class, f); + + // !!! if the type is the same and the target is "*", then it's good to merge + if (upgradePack.isAllTarget() && upgradePack.getType() == base.getType()) { + return upgradePack; + } + + } catch (Exception e) { + LOG.warn("File {} does not appear to be an upgrade pack and will be skipped ({})", + f.getAbsolutePath(), e.getMessage()); + } + } + + return null; + } + + /** * Parses the service specific upgrade file and merges the none order elements * (prerequisite check and processing sections). */ @@ -1011,12 +1080,24 @@ public class StackModule extends BaseModule implements V throw new AmbariException("Unable to parse service upgrade file at location: " + serviceFile.getAbsolutePath(), e); } - parent.mergePrerequisiteChecks(pack); - parent.mergeProcessing(pack); + return parseServiceUpgradePack(parent, pack); + } - return pack; + /** + * Places prerequisite checks and processing objects onto the parent upgrade pack. + * + * @param parent the parent upgrade pack + * @param child the parsed child upgrade pack + * @return the child upgrade pack + */ + private UpgradePack parseServiceUpgradePack(UpgradePack parent, UpgradePack child) { + parent.mergePrerequisiteChecks(child); + parent.mergeProcessing(child); + + return child; } + /** * Process repositories associated with the stack. * @throws AmbariException if unable to fully process the stack repositories http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java index edf5c89..aa97fa3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java @@ -52,6 +52,8 @@ import org.slf4j.LoggerFactory; @XmlAccessorType(XmlAccessType.FIELD) public class UpgradePack { + private static final String ALL_VERSIONS = "*"; + private static Logger LOG = LoggerFactory.getLogger(UpgradePack.class); /** @@ -315,7 +317,6 @@ public class UpgradePack { public boolean canBeApplied(String targetVersion){ // check that upgrade pack can be applied to selected stack // converting 2.2.*.* -> 2\.2(\.\d+)?(\.\d+)?(-\d+)? - String regexPattern = getTarget().replaceAll("\\.", "\\\\."); // . -> \. regexPattern = regexPattern.replaceAll("\\\\\\.\\*", "(\\\\\\.\\\\d+)?"); // \.* -> (\.\d+)? regexPattern = regexPattern.concat("(-\\d+)?"); @@ -453,6 +454,15 @@ public class UpgradePack { } /** + * @return {@code true} if the upgrade targets any version or stack. Both + * {@link #target} and {@link #targetStack} must equal "*" + */ + public boolean isAllTarget() { + return ALL_VERSIONS.equals(target) && ALL_VERSIONS.equals(targetStack); + } + + + /** * A service definition that holds a list of components in the 'order' element. */ public static class OrderService { http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index a838fc7..e675633 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -421,7 +421,7 @@ - + UPGRADE false true @@ -443,7 +443,7 @@ - + DOWNGRADE false true http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index a01996a..7d4cfc9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -422,7 +422,7 @@ - + UPGRADE false true @@ -444,7 +444,7 @@ - + DOWNGRADE false true http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index 17be4a8..effdda2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -406,7 +406,7 @@ - + UPGRADE false true @@ -428,7 +428,7 @@ - + DOWNGRADE false true http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index f520faf..31e72a1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -411,7 +411,7 @@ - + UPGRADE false true @@ -433,7 +433,7 @@ - + DOWNGRADE false true http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/upgrade-pack.xsd ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/upgrade-pack.xsd b/ambari-server/src/main/resources/upgrade-pack.xsd index 2871f05..45cf5fa 100644 --- a/ambari-server/src/main/resources/upgrade-pack.xsd +++ b/ambari-server/src/main/resources/upgrade-pack.xsd @@ -392,7 +392,15 @@ - + + + + Ensures that the element "processing" does not have duplicate services + + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java index dde7ffa..8f9f791 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.state.stack; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -570,6 +571,27 @@ public class UpgradePackTest { Map> tasks = upgradePack.getTasks(); assertTrue(tasks.containsKey("HBASE")); + + // !!! generalized upgrade pack shouldn't be in this + boolean found = false; + for (Grouping grouping : upgradePack.getAllGroups()) { + if (grouping.name.equals("GANGLIA_UPGRADE")) { + found = true; + break; + } + } + assertFalse(found); + + // !!! test merge of a generalized upgrade pack + upgradePack = upgrades.get("upgrade_test_conditions"); + assertNotNull(upgradePack); + for (Grouping grouping : upgradePack.getAllGroups()) { + if (grouping.name.equals("GANGLIA_UPGRADE")) { + found = true; + break; + } + } + assertTrue(found); } http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml new file mode 100644 index 0000000..c2224dd --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml @@ -0,0 +1,27 @@ + + + + * + * + ROLLING + + + ZOOKEEPER_CONFIG_CONDITION_TEST + true + true + + HST_SERVER + + + + + + + + + + + + + + \ No newline at end of file