ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-13824. Stop-and-Start Upgrade: Fix for Oozie Express upgrade issues from HDP 2.1 to 2.3 (Swapan Shridhar via alejandro)
Date Thu, 12 Nov 2015 19:36:36 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 14ef5e6bf -> 733457d95


AMBARI-13824. Stop-and-Start Upgrade: Fix for Oozie Express upgrade issues from HDP 2.1 to
2.3 (Swapan Shridhar via alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/733457d9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/733457d9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/733457d9

Branch: refs/heads/branch-2.1
Commit: 733457d95d3d625d783938c75975e61dbe8187e4
Parents: 14ef5e6
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Thu Nov 12 11:35:49 2015 -0800
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Thu Nov 12 11:35:49 2015 -0800

----------------------------------------------------------------------
 .../libraries/functions/conf_select.py          | 31 +++++++++----
 .../AmbariCustomCommandExecutionHelper.java     | 37 +++++++++++++--
 .../internal/UpgradeResourceProvider.java       | 47 ++++++++++----------
 .../state/stack/upgrade/ConfigureFunction.java  | 36 +++++++++++++++
 .../server/state/stack/upgrade/Grouping.java    |  3 ++
 .../state/stack/upgrade/StageWrapper.java       |  7 +--
 .../ambari/server/state/stack/upgrade/Task.java |  8 +++-
 .../4.0.0.2.0/package/scripts/oozie_server.py   | 20 +++++++--
 .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml |  7 ++-
 9 files changed, 153 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index e2ec6ad..6b7d09d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -224,14 +224,12 @@ def create(stack_name, package, version, dry_run = False):
 
 def select(stack_name, package, version, try_create=True):
   """
-  Selects a config version for the specified package.  Currently only works if the version
is
-  for HDP-2.3 or higher
+  Selects a config version for the specified package.
   :stack_name: the name of the stack
   :package: the name of the package, as-used by conf-select
   :version: the version number to create
   :try_create: optional argument to attempt to create the directory before setting it
   """
-
   if not _valid(stack_name, package, version):
     return
 
@@ -240,6 +238,28 @@ def select(stack_name, package, version, try_create=True):
 
   shell.checked_call(get_cmd("set-conf-dir", package, version), logoutput=False, quiet=False,
sudo=True)
 
+  # Create the symbolic link using 'PACKAGE_DIRS' for the given package
+  # Starting with 2.3, we have sym links instead of flat directories.
+  # Eg: /etc/<service-name>/conf -> /etc/<service-name>/2.3.x.y-<version>/0
+  # But, in case of Express upgrade from HDP 2.1-> HDP 2.3 where we have
+  # deleted the /etc/<service-name>/conf directory, the above mentioned
+  # symlink needs to be created here.
+  if package in PACKAGE_DIRS:
+    conf_dirs = PACKAGE_DIRS[package]
+    Logger.info("For package : {0}, DIRS = {1}".format(package, conf_dirs))
+    for dirInfo in conf_dirs:
+      if "conf_dir" in dirInfo and "current_dir" in dirInfo:
+        conf_dir = dirInfo["conf_dir"]
+        current_dir = dirInfo["current_dir"]
+        Logger.info("For package : {0}, Source dir: {1}, Dest dir: {2}".format(package, conf_dir,
current_dir))
+        if os.path.exists(current_dir):
+          real_path_of_current_dir = os.path.realpath(current_dir)
+          normalized_conf_dir = (os.path.normpath(conf_dir)).strip()
+          normalized_current_dir = (os.path.normpath(real_path_of_current_dir)).strip()
+          Logger.info("Normalized Conf Dir : {0}, Normalized Current Dir : {1}".format(normalized_conf_dir,
normalized_current_dir))
+          if not os.path.isdir(normalized_conf_dir) and os.path.isdir(normalized_current_dir)
and normalized_current_dir != normalized_conf_dir:
+            Logger.info("Creating Symlink : {0} -> {1}".format(normalized_conf_dir, normalized_current_dir))
+            os.symlink(normalized_current_dir, normalized_conf_dir)
 
 def get_hadoop_conf_dir(force_latest_on_upgrade=False):
   """
@@ -293,23 +313,18 @@ def create_config_links(stack_id, stack_version):
   stack_id:  stack id, ie HDP-2.3
   stack_version:  version to set, ie 2.3.0.0-1234
   """
-
   if stack_id is None:
     Logger.info("Cannot create config links when stack_id is not defined")
     return
-
   args = stack_id.upper().split('-')
   if len(args) != 2:
     Logger.info("Unrecognized stack id {0}".format(stack_id))
     return
-
   if args[0] != "HDP":
     Logger.info("Unrecognized stack name {0}".format(args[0]))
-
   if version.compare_versions(version.format_hdp_stack_version(args[1]), "2.3.0.0") <
0:
     Logger.info("Cannot link configs unless HDP-2.3 or higher")
     return
-
   for k, v in PACKAGE_DIRS.iteritems():
     dirs = create(args[0], k, stack_version, dry_run = True)
     if 0 == len(dirs):

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index c515e46..6fad6f9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -179,10 +179,41 @@ public class AmbariCustomCommandExecutionHelper {
         stackId.getStackName(), stackId.getStackVersion(),
         serviceName, componentName);
 
-    return !(!componentInfo.isCustomCommand(commandName) &&
-      !actionMetadata.isDefaultHostComponentCommand(commandName));
+    return actionMetadata.isDefaultHostComponentCommand(commandName);
   }
 
+  private Boolean isValidDefaultCommand(String clusterName,
+      String serviceName, String componentName, String commandName)
+      throws AmbariException {
+
+      Cluster cluster = clusters.getCluster(clusterName);
+      StackId stackId = cluster.getDesiredStackVersion();
+
+      if (componentName == null) {
+        return false;
+      }
+      ComponentInfo componentInfo = ambariMetaInfo.getComponent(
+        stackId.getStackName(), stackId.getStackVersion(),
+        serviceName, componentName);
+
+      return (actionMetadata.isDefaultHostComponentCommand(commandName));
+    }
+
+    private Boolean isValidDefaultCommand(ActionExecutionContext
+     actionExecutionContext, RequestResourceFilter resourceFilter)
+     throws AmbariException {
+      String clusterName = actionExecutionContext.getClusterName();
+      String serviceName = resourceFilter.getServiceName();
+      String componentName = resourceFilter.getComponentName();
+      String commandName = actionExecutionContext.getActionName();
+
+      if (componentName == null) {
+        return false;
+      }
+
+      return isValidCustomCommand(clusterName, serviceName, componentName, commandName);
+    }
+
   private Boolean isValidCustomCommand(ActionExecutionContext
       actionExecutionContext, RequestResourceFilter resourceFilter)
       throws AmbariException {
@@ -896,7 +927,7 @@ public class AmbariCustomCommandExecutionHelper {
         findHostAndAddServiceCheckAction(actionExecutionContext, resourceFilter, stage);
       } else if (actionName.equals(DECOMMISSION_COMMAND_NAME)) {
         addDecommissionAction(actionExecutionContext, resourceFilter, stage);
-      } else if (isValidCustomCommand(actionExecutionContext, resourceFilter)) {
+      } else if (isValidDefaultCommand(actionExecutionContext, resourceFilter)) {
 
         String commandDetail = getReadableCustomCommandDetail(actionExecutionContext, resourceFilter);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 715a089..0d99b78 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -17,25 +17,10 @@
  */
 package org.apache.ambari.server.controller.internal;
 
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
-import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
-
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.Role;
 import org.apache.ambari.server.RoleCommand;
@@ -108,10 +93,24 @@ import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Lists;
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION;
 
 /**
  * Manages the ability to start and get status of upgrades.
@@ -1095,6 +1094,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
           throws AmbariException {
 
     switch (wrapper.getType()) {
+      case CONFIGURE:
       case START:
       case STOP:
       case RESTART:
@@ -1242,6 +1242,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
 
     String function = null;
     switch (wrapper.getType()) {
+      case CONFIGURE:
       case START:
       case STOP:
       case RESTART:

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java
new file mode 100644
index 0000000..fc6a45b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureFunction.java
@@ -0,0 +1,36 @@
+package org.apache.ambari.server.state.stack.upgrade;
+
+  import javax.xml.bind.annotation.XmlAccessType;
+  import javax.xml.bind.annotation.XmlAccessorType;
+  import javax.xml.bind.annotation.XmlRootElement;
+  import javax.xml.bind.annotation.XmlTransient;
+  import javax.xml.bind.annotation.XmlType;
+
+/**
+ * Used to represent Configuring of a component.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name="configure_function")
+public class ConfigureFunction extends Task {
+
+  @XmlTransient
+  private Task.Type type = Type.CONFIGURE_FUNCTION;
+
+  public static final String actionVerb = "Configuring";
+
+  @Override
+  public Task.Type getType() {
+    return type;
+  }
+
+  @Override
+  public StageWrapper.Type getStageWrapperType() {
+    return StageWrapper.Type.CONFIGURE;
+  }
+
+  @Override
+  public String getActionVerb() {
+    return actionVerb;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
index 4a94dc1..65115f6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
@@ -246,6 +246,9 @@ public class Grouping {
         case EXECUTE:
           type = StageWrapper.Type.RU_TASKS;
           break;
+        case CONFIGURE_FUNCTION:
+          type = StageWrapper.Type.CONFIGURE;
+          break;
         case RESTART:
           type = StageWrapper.Type.RESTART;
           break;

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
index 2ea3671..86a8f55 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapper.java
@@ -17,6 +17,8 @@
  */
 package org.apache.ambari.server.state.stack.upgrade;
 
+import com.google.gson.Gson;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -25,8 +27,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.google.gson.Gson;
-
 /**
  *
  */
@@ -149,6 +149,7 @@ public class StageWrapper {
     RU_TASKS,
     SERVICE_CHECK,
     STOP,
-    START
+    START,
+    CONFIGURE
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
index f443e53..60091a6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
@@ -24,7 +24,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
 /**
  * Base class to identify the items that could possibly occur during an upgrade
  */
-@XmlSeeAlso(value={ExecuteTask.class, ConfigureTask.class, ManualTask.class, RestartTask.class,
StartTask.class, StopTask.class, ServerActionTask.class})
+@XmlSeeAlso(value={ExecuteTask.class, ConfigureTask.class, ManualTask.class, RestartTask.class,
StartTask.class, StopTask.class, ServerActionTask.class, ConfigureFunction.class})
 public abstract class Task {
 
   /**
@@ -66,6 +66,10 @@ public abstract class Task {
      */
     CONFIGURE,
     /**
+     * Task that sets up the configuration for subsequent task
+     */
+    CONFIGURE_FUNCTION,
+    /**
      * Task that displays a message and must be confirmed before continuing
      */
     MANUAL,
@@ -101,7 +105,7 @@ public abstract class Task {
      * @return {@code true} if the task is a command type (as opposed to an action)
      */
     public boolean isCommand() {
-      return this == RESTART || this == START || this == STOP || this == SERVICE_CHECK;
+      return this == RESTART || this == START || this == CONFIGURE_FUNCTION || this == STOP
|| this == SERVICE_CHECK;
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
index f35df93..35975df 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
@@ -24,6 +24,7 @@ from resource_management.libraries.functions import compare_versions
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import hdp_select
 from resource_management.libraries.functions import format_hdp_stack_version
+from resource_management.libraries.functions.constants import Direction
 from resource_management.libraries.functions.security_commons import build_expectations
 from resource_management.libraries.functions.security_commons import cached_kinit_executor
 from resource_management.libraries.functions.security_commons import get_params_from_filesystem
@@ -47,8 +48,20 @@ class OozieServer(Script):
   def install(self, env):
     self.install_packages(env)
 
-  def configure(self, env):
+  def configure(self, env, upgrade_type=None):
     import params
+
+    if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and
\
+            params.version and compare_versions(format_hdp_stack_version(params.version),
'2.2.0.0') >= 0:
+      conf_select.select(params.stack_name, "oozie", params.version)
+      # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new
version of
+      # oozie, we need to create the symlinks both for server and client.
+      # This is required as both need to be pointing to new installed oozie version.
+
+      # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie
+      hdp_select.select("oozie-client", params.version)
+      # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie
+      hdp_select.select("oozie-server", params.version)
     env.set_params(params)
 
     oozie(is_server=True)
@@ -166,8 +179,9 @@ class OozieServerDefault(OozieServer):
 
     OozieUpgrade.backup_configuration()
 
-    conf_select.select(params.stack_name, "oozie", params.version)
-    hdp_select.select("oozie-server", params.version)
+    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
+      conf_select.select(params.stack_name, "oozie", params.version)
+      hdp_select.select("oozie-server", params.version)
 
     OozieUpgrade.restore_configuration()
     OozieUpgrade.prepare_libext_directory()

http://git-wip-us.apache.org/repos/asf/ambari/blob/733457d9/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index f12d0cd..49c8385 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -697,6 +697,11 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
+          <!-- We need to set up the "/etc/oozie/conf" symlink before upgrading, as the
HDP 2.1 directory
+          pertaining to oozie has been deleted as part of HDP 2.1 removal in Express Upgrade
+          from HDP 2.1->2.3.
+          -->
+          <task xsi:type="configure_function"/>
           <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating
a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
@@ -775,4 +780,4 @@
       </component>
     </service>
   </processing>
-</upgrade>
\ No newline at end of file
+</upgrade>


Mime
View raw message