ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [01/50] [abbrv] ambari git commit: AMBARI-18937. Allow generic Service Upgrade Packs for all targets (ncole)
Date Mon, 28 Nov 2016 19:22:55 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-dev-patch-upgrade efcfced54 -> 1f8b527f9


AMBARI-18937. Allow generic Service Upgrade Packs for all targets (ncole)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7d07b71d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7d07b71d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7d07b71d

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 7d07b71de7019efaaa04fe8c09548cdc604cea60
Parents: ef91365
Author: Nate Cole <ncole@hortonworks.com>
Authored: Mon Nov 21 15:24:59 2016 -0500
Committer: Nate Cole <ncole@hortonworks.com>
Committed: Mon Nov 21 16:11:21 2016 -0500

----------------------------------------------------------------------
 .../apache/ambari/server/stack/StackModule.java | 101 +++++++++++++++++--
 .../ambari/server/state/stack/UpgradePack.java  |  12 ++-
 .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml     |   4 +-
 .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml     |   4 +-
 .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml     |   4 +-
 .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml     |   4 +-
 .../src/main/resources/upgrade-pack.xsd         |  10 +-
 .../server/state/stack/UpgradePackTest.java     |  22 ++++
 .../GANGLIA/upgrades/HDP/rolling-upgrade.xml    |  27 +++++
 9 files changed, 168 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index d9eaf27..ce606de 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -46,16 +46,19 @@ import org.apache.ambari.server.state.stack.ServiceMetainfoXml;
 import org.apache.ambari.server.state.stack.StackMetainfoXml;
 import org.apache.ambari.server.state.stack.UpgradePack;
 import org.apache.ambari.server.state.stack.upgrade.Grouping;
+import org.apache.ambari.server.state.stack.upgrade.UpgradeType;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Function;
-import com.google.common.collect.ListMultimap;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimaps;
 
+
 /**
  * Stack module which provides all functionality related to parsing and fully
  * resolving stacks from the stack definition.
@@ -834,6 +837,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
     if (configPack == null) {
       return;
     }
+
     for (ServiceModule module : serviceModules.values()) {
       File upgradesFolder = module.getModuleInfo().getServiceUpgradesFolder();
       if (upgradesFolder != null) {
@@ -869,12 +873,17 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
   private UpgradePack getServiceUpgradePack(UpgradePack pack, File upgradesFolder) throws
AmbariException {
     File stackFolder = new File(upgradesFolder, stackInfo.getName());
     File versionFolder = new File(stackFolder, stackInfo.getVersion());
+    // !!! relies on the service upgrade pack filename being named the exact same
     File servicePackFile = new File(versionFolder, pack.getName() + ".xml");
+
     LOG.info("Service folder: " + servicePackFile.getAbsolutePath());
-    if (!servicePackFile.exists()) {
-      return null;
+    if (servicePackFile.exists()) {
+      return parseServiceUpgradePack(pack, servicePackFile);
+    } else {
+      UpgradePack child = findServiceUpgradePack(pack, stackFolder);
+
+      return null == child ? null : parseServiceUpgradePack(pack, child);
     }
-    return parseServiceUpgradePack(pack, servicePackFile);
   }
 
   /**
@@ -882,6 +891,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
    * for any service which specifies its own upgrade.
    */
   private void mergeUpgradePack(UpgradePack pack, List<UpgradePack> servicePacks) throws
AmbariException {
+
     List<Grouping> originalGroups = pack.getAllGroups();
     Map<String, List<Grouping>> allGroupMap = new HashMap<>();
     for (Grouping group : originalGroups) {
@@ -889,8 +899,21 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
       list.add(group);
       allGroupMap.put(group.name, list);
     }
+
     for (UpgradePack servicePack : servicePacks) {
       for (Grouping group : servicePack.getAllGroups()) {
+
+        /*
+         !!! special case where the service pack is targeted for any version.  When
+         a service UP targets to run after another group, check to make sure that the
+         base UP contains the group.
+         */
+        if (servicePack.isAllTarget() && !allGroupMap.keySet().contains(group.addAfterGroup))
{
+          LOG.warn("Service Upgrade Pack specified after-group of {}, but that is not found
in {}",
+              group.addAfterGroup, StringUtils.join(allGroupMap.keySet(), ','));
+          continue;
+        }
+
         if (allGroupMap.containsKey(group.name)) {
           List<Grouping> list = allGroupMap.get(group.name);
           Grouping first = list.get(0);
@@ -906,8 +929,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
           else {
             list.add(group);
           }
-        }
-        else {
+        } else {
           List<Grouping> list = new ArrayList<>();
           list.add(group);
           allGroupMap.put(group.name, list);
@@ -934,15 +956,17 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
    */
   private void orderGroups(List<Grouping> groups, Map<String, Grouping> mergedGroupMap)
throws AmbariException {
     Map<String, List<Grouping>> skippedGroups = new HashMap<>();
+
     for (Map.Entry<String, Grouping> entry : mergedGroupMap.entrySet()) {
-      String key = entry.getKey();
       Grouping group = entry.getValue();
+
       if (!groups.contains(group)) {
         boolean added = addGrouping(groups, group);
         if (added) {
           addSkippedGroup(groups, skippedGroups, group);
         } else {
           List<Grouping> tmp = null;
+
           // store the group until later
           if (skippedGroups.containsKey(group.addAfterGroup)) {
             tmp = skippedGroups.get(group.addAfterGroup);
@@ -954,6 +978,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
         }
       }
     }
+
     if (!skippedGroups.isEmpty()) {
       throw new AmbariException("Missing groups: " + skippedGroups.keySet());
     }
@@ -999,6 +1024,50 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
   }
 
   /**
+   * Finds an upgrade pack that:
+   * <ul>
+   *   <li>Is found in the $SERVICENAME/upgrades/$STACKNAME folder</li>
+   *   <li>Matches the same {@link UpgradeType#getType()}as the {@code base} upgrade
pack</li>
+   *   <li>Has the {@link UpgradePack#getTarget()} value equals to "*"</li>
+   *   <li>Has the {@link UpgradePack#getTargetStack()} value equals to "*"</li>
+   * </ul>
+   * This method will not attempt to resolve the "most correct" upgrade pack.  For this
+   * feature to work, there should be only one upgrade pack per type.  If more specificity
+   * is required, then follow the convention of $SERVICENAME/upgrades/$STACKNAME/$STACKVERSION/$BASE_FILE_NAME.xml
+   *
+   * @param base the base upgrade pack for a stack
+   * @param upgradeStackDirectory service directory that contains stack upgrade files.
+   * @return an upgrade pack that matches {@code base}
+   */
+  private UpgradePack findServiceUpgradePack(UpgradePack base, File upgradeStackDirectory)
{
+    if (!upgradeStackDirectory.exists() || !upgradeStackDirectory.isDirectory()) {
+      return null;
+    }
+
+    File[] upgradeFiles = upgradeStackDirectory.listFiles(StackDirectory.XML_FILENAME_FILTER);
+    if (0 == upgradeFiles.length) {
+      return null;
+    }
+
+    for (File f : upgradeFiles) {
+      try {
+        UpgradePack upgradePack = unmarshaller.unmarshal(UpgradePack.class, f);
+
+        // !!! if the type is the same and the target is "*", then it's good to merge
+        if (upgradePack.isAllTarget() && upgradePack.getType() == base.getType())
{
+          return upgradePack;
+        }
+
+      } catch (Exception e) {
+        LOG.warn("File {} does not appear to be an upgrade pack and will be skipped ({})",
+            f.getAbsolutePath(), e.getMessage());
+      }
+    }
+
+    return null;
+  }
+
+  /**
    * Parses the service specific upgrade file and merges the none order elements
    * (prerequisite check and processing sections).
    */
@@ -1011,12 +1080,24 @@ public class StackModule extends BaseModule<StackModule, StackInfo>
implements V
       throw new AmbariException("Unable to parse service upgrade file at location: " + serviceFile.getAbsolutePath(),
e);
     }
 
-    parent.mergePrerequisiteChecks(pack);
-    parent.mergeProcessing(pack);
+    return parseServiceUpgradePack(parent, pack);
+  }
 
-    return pack;
+  /**
+   * Places prerequisite checks and processing objects onto the parent upgrade pack.
+   *
+   * @param parent  the parent upgrade pack
+   * @param child   the parsed child upgrade pack
+   * @return the child upgrade pack
+   */
+  private UpgradePack parseServiceUpgradePack(UpgradePack parent, UpgradePack child) {
+    parent.mergePrerequisiteChecks(child);
+    parent.mergeProcessing(child);
+
+    return child;
   }
 
+
   /**
    * Process repositories associated with the stack.
    * @throws AmbariException if unable to fully process the stack repositories

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
index edf5c89..aa97fa3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java
@@ -52,6 +52,8 @@ import org.slf4j.LoggerFactory;
 @XmlAccessorType(XmlAccessType.FIELD)
 public class UpgradePack {
 
+  private static final String ALL_VERSIONS = "*";
+
   private static Logger LOG = LoggerFactory.getLogger(UpgradePack.class);
 
   /**
@@ -315,7 +317,6 @@ public class UpgradePack {
   public boolean canBeApplied(String targetVersion){
     // check that upgrade pack can be applied to selected stack
     // converting 2.2.*.* -> 2\.2(\.\d+)?(\.\d+)?(-\d+)?
-
     String regexPattern = getTarget().replaceAll("\\.", "\\\\."); // . -> \.
     regexPattern = regexPattern.replaceAll("\\\\\\.\\*", "(\\\\\\.\\\\d+)?"); // \.* ->
(\.\d+)?
     regexPattern = regexPattern.concat("(-\\d+)?");
@@ -453,6 +454,15 @@ public class UpgradePack {
   }
 
   /**
+   * @return {@code true} if the upgrade targets any version or stack.  Both
+   * {@link #target} and {@link #targetStack} must equal "*"
+   */
+  public boolean isAllTarget() {
+    return ALL_VERSIONS.equals(target) && ALL_VERSIONS.equals(targetStack);
+  }
+
+
+  /**
    * A service definition that holds a list of components in the 'order' element.
    */
   public static class OrderService {

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index a838fc7..e675633 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -421,7 +421,7 @@
       </execute-stage>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
       <direction>UPGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -443,7 +443,7 @@
       </service>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
       <direction>DOWNGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index a01996a..7d4cfc9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -422,7 +422,7 @@
       </execute-stage>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
       <direction>UPGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -444,7 +444,7 @@
       </service>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
       <direction>DOWNGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 17be4a8..effdda2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -406,7 +406,7 @@
       </execute-stage>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
       <direction>UPGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -428,7 +428,7 @@
       </service>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
       <direction>DOWNGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index f520faf..31e72a1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -411,7 +411,7 @@
       </execute-stage>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_UPGRADE" title="Stop Storm Services">
       <direction>UPGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -433,7 +433,7 @@
       </service>
     </group>
 
-    <group xsi:type="stop" name="STOP_STORM" title="Stop Storm Services">
+    <group xsi:type="stop" name="STOP_STORM_DOWNGRADE" title="Stop Storm Services">
       <direction>DOWNGRADE</direction>
       <service-check>false</service-check>
       <skippable>true</skippable>

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/main/resources/upgrade-pack.xsd
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade-pack.xsd b/ambari-server/src/main/resources/upgrade-pack.xsd
index 2871f05..45cf5fa 100644
--- a/ambari-server/src/main/resources/upgrade-pack.xsd
+++ b/ambari-server/src/main/resources/upgrade-pack.xsd
@@ -392,7 +392,15 @@
         <xs:element name="type" type="upgrade-kind-type" />
         <xs:element name="prerequisite-checks" type="prerequisite-check-type" minOccurs="0"
/>
         <xs:element name="upgrade-path" type="upgrade-path-type" minOccurs="0" /> 
-        <xs:element name="order" type="order-type" />
+        <xs:element name="order" type="order-type">
+          <xs:unique name="unique-by-group">
+            <xs:annotation>
+              <xs:documentation>Ensures that the element "processing" does not have
duplicate services</xs:documentation>
+            </xs:annotation>
+            <xs:selector xpath="group" />
+            <xs:field xpath="@name" />
+          </xs:unique>
+        </xs:element>
         <xs:element name="processing" type="processing-type">
           <xs:unique name="unique-by-service">
             <xs:annotation>

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
index dde7ffa..8f9f791 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackTest.java
@@ -18,6 +18,7 @@
 package org.apache.ambari.server.state.stack;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -570,6 +571,27 @@ public class UpgradePackTest {
 
     Map<String, Map<String, ProcessingComponent>> tasks = upgradePack.getTasks();
     assertTrue(tasks.containsKey("HBASE"));
+
+    // !!! generalized upgrade pack shouldn't be in this
+    boolean found = false;
+    for (Grouping grouping : upgradePack.getAllGroups()) {
+      if (grouping.name.equals("GANGLIA_UPGRADE")) {
+        found = true;
+        break;
+      }
+    }
+    assertFalse(found);
+
+    // !!! test merge of a generalized upgrade pack
+    upgradePack = upgrades.get("upgrade_test_conditions");
+    assertNotNull(upgradePack);
+    for (Grouping grouping : upgradePack.getAllGroups()) {
+      if (grouping.name.equals("GANGLIA_UPGRADE")) {
+        found = true;
+        break;
+      }
+    }
+    assertTrue(found);
   }
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7d07b71d/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml
b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml
new file mode 100644
index 0000000..c2224dd
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+
+<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-pack.xsd">
+  <target>*</target>
+  <target-stack>*</target-stack>
+  <type>ROLLING</type>
+  <order>
+    <group name="GANGLIA_UPGRADE" title="Ganglia">
+      <add-after-group>ZOOKEEPER_CONFIG_CONDITION_TEST</add-after-group>
+      <skippable>true</skippable>
+      <allow-retry>true</allow-retry>
+      <service name="GANGLIA">
+        <component>HST_SERVER</component>
+      </service>
+    </group>
+  </order>
+
+  <processing>
+    <service name="GANGLIA">
+      <component name="GANGLIA_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+    </service>
+  </processing>
+</upgrade>
\ No newline at end of file


Mime
View raw message