ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [22/51] [abbrv] ambari git commit: AMBARI-22337 each service should be able to implement server actions, package them add a jar to be loaded during EU (dili)
Date Mon, 13 Nov 2017 16:57:39 GMT
AMBARI-22337 each service should be able to implement server actions, package them add a jar
to be loaded during EU (dili)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/063ba36e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/063ba36e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/063ba36e

Branch: refs/heads/branch-feature-AMBARI-21674
Commit: 063ba36ef5d2ae2f407b46aae3f993ab81eae310
Parents: 38f6740
Author: Di Li <dili@apache.org>
Authored: Tue Nov 7 10:51:48 2017 -0500
Committer: Di Li <dili@apache.org>
Committed: Tue Nov 7 10:51:48 2017 -0500

----------------------------------------------------------------------
 ambari-server/pom.xml                           |  12 ++
 .../serveraction/ServerActionExecutor.java      | 147 +++++++++++++++++--
 .../ambari/server/stack/ServiceDirectory.java   |  29 ++++
 .../ambari/server/stack/ServiceModule.java      |   8 +
 .../apache/ambari/server/state/ServiceInfo.java |  14 ++
 .../ambari/server/stack/ServiceModuleTest.java  |  30 ++++
 .../server/stack/StackManagerExtensionTest.java |   6 +
 7 files changed, 232 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index a86acf5..2ecf435 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -198,6 +198,18 @@
               </target>
             </configuration>
           </execution>
+          <execution>
+            <id>generate-test-oozie2-server-actions-dir</id>
+            <phase>process-test-classes</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <mkdir dir="target/test-classes/extensions/EXT/0.1/services/OOZIE2/server_actions/tmp"/>
+              </target>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
       <plugin>

http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
index 50e3cfe..e219dc3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
@@ -18,10 +18,17 @@
 
 package org.apache.ambari.server.serveraction;
 
+import java.io.File;
+import java.io.FilenameFilter;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.StringTokenizer;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
@@ -38,11 +45,17 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.actionmanager.Stage;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.UpgradeContext.UpgradeServiceSummary;
+import org.apache.ambari.server.state.UpgradeContext.UpgradeSummary;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.util.ClassUtils;
 
 import com.google.inject.Inject;
 import com.google.inject.Injector;
@@ -495,7 +508,6 @@ public class ServerActionExecutor {
         throw new AmbariException("Missing ExecutionCommand data");
       } else {
         Map<String, String> roleParams = executionCommand.getRoleParams();
-
         if (roleParams == null) {
           throw new AmbariException("Missing RoleParams data");
         } else {
@@ -504,8 +516,30 @@ public class ServerActionExecutor {
           if (actionClassname == null) {
             throw new AmbariException("Missing action classname for server action");
           } else {
-            ServerAction action = createServerAction(actionClassname);
-
+            Map<String, ServiceInfo> services = new HashMap<String, ServiceInfo>();
+            UpgradeSummary upgradeSummary = executionCommand.getUpgradeSummary();
+            if (upgradeSummary != null) {
+              Map<String, UpgradeServiceSummary> upgradeServiceSummaries = upgradeSummary.services;
+              LOG.debug("UpgradeServiceSummary: " + upgradeServiceSummaries);
+              AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+              AmbariMetaInfo ambariMetaInfo = ambariManagementController.getAmbariMetaInfo();
+              String serviceName = executionCommand.getServiceName();
+              if (serviceName != null && !serviceName.isEmpty()){
+                LOG.info(String.format("Server action %s is associated with service %s",
actionClassname, serviceName));
+                //Execution stage of a given service, only need to examine stack information
for this one service
+                UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(serviceName);
+                addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId, serviceName);
+              } else {
+                LOG.info(String.format("Server action %s is not associated with a service",
actionClassname));
+                //Load all Jars
+                for(String key: upgradeServiceSummaries.keySet()){
+                  UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(key);
+                  addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId,
key);
+                }
+              }
+              LOG.info(String.format("Attempt to load server action classes from %s", services.keySet().toString()));
+            }
+            ServerAction action = createServerAction(actionClassname, services);
             if (action == null) {
               throw new AmbariException("Failed to create server action: " + actionClassname);
             } else {
@@ -520,6 +554,30 @@ public class ServerActionExecutor {
       }
     }
 
+    private void addServiceInfo(Map<String, ServiceInfo> services, AmbariMetaInfo ambariMetaInfo,
String stackId, String serviceName) {
+      List<String> stackInfo = getStackInfo(stackId);
+      LOG.debug(String.format("Stack info list: %s", stackInfo));
+      if (stackInfo.size() > 1) {
+        try {
+          ServiceInfo service = ambariMetaInfo.getService(stackInfo.get(0), stackInfo.get(1),
serviceName);
+          LOG.debug(String.format("Adding %s to the list of services for loading external
Jars...", service.getName()));
+          services.put(serviceName, service);
+        } catch (AmbariException e) {
+          LOG.error(String.format("Failed to obtain service info for stack %s, service name
%s", stackId, serviceName), e);
+        }
+      }
+    }
+
+    private List<String> getStackInfo(String stackId) {
+      LOG.debug(String.format("Stack id: %s", stackId));
+      StringTokenizer tokens = new StringTokenizer(stackId, "-");
+      List<String> info = new ArrayList<String>();
+      while (tokens.hasMoreElements()) {
+        info.add((String)tokens.nextElement());
+      }
+      return info;
+    }
+
     /**
      * Attempts to create an instance of the ServerAction class implementation specified
in
      * classname.
@@ -528,24 +586,85 @@ public class ServerActionExecutor {
      * @return the instantiated ServerAction implementation
      * @throws AmbariException
      */
-    private ServerAction createServerAction(String classname) throws AmbariException {
-      try {
-        Class<?> actionClass = Class.forName(classname);
+    private ServerAction createServerAction(String classname, Map<String, ServiceInfo>
services) throws AmbariException {
+      Class<?> actionClass = null;
+      actionClass = getServerActionClass(classname);
+      if (actionClass == null) {
+        LOG.debug(String.format("Did not find %s in Ambari, try to load it from external
directories", classname));
+        actionClass = getServiceLevelServerActionClass(classname, services);
+      }
 
-        if (actionClass == null) {
-          throw new AmbariException("Unable to load server action class: " + classname);
+      if (actionClass == null) {
+        throw new AmbariException("Unable to load server action class: " + classname);
+      } else {
+        LOG.debug(String.format("Ready to init server action %s", classname));
+        Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class);
+        if (serverActionClass == null) {
+          throw new AmbariException("Unable to execute server action class, invalid type:
" + classname);
         } else {
-          Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class);
+          return injector.getInstance(serverActionClass);
+        }
+      }
+    }
 
-          if (serverActionClass == null) {
-            throw new AmbariException("Unable to execute server action class, invalid type:
" + classname);
-          } else {
-            return injector.getInstance(serverActionClass);
+    /**
+     * Load server action classes defined in the service level Jar files
+     * */
+    private Class<?> getServiceLevelServerActionClass(String classname, Map<String,
ServiceInfo> services) {
+      List<URL> urls = new ArrayList<>();
+      for (ServiceInfo service : services.values()) {
+        LOG.debug(String.format("Checking service %s", service));
+        File dir = service.getServerActionsFolder();
+        if ( dir != null) {
+          LOG.debug(String.format("Service %s, external dir %s",service.getName(), dir.getAbsolutePath()));
+          File[] jars = dir.listFiles(new FilenameFilter() {
+            @Override
+            public boolean accept(File dir, String name) {
+              LOG.debug(String.format("Checking folder %s", name));
+              return name.endsWith(".jar");
+            }
+          });
+          for (File jar : jars) {
+            try {
+              URL url = jar.toURI().toURL();
+              urls.add(url);
+              LOG.info("Adding server action jar to classpath: {}", url);
+            }
+            catch (Exception e) {
+              LOG.error("Failed to add server action jar to classpath: {}", jar.getAbsolutePath(),
e);
+            }
           }
+        } else {
+          LOG.error(String.format("%s service server actions folder returned null", service));
+        }
+      }
+
+      ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), ClassUtils.getDefaultClassLoader());
+      Class<?> actionClass = null;
+      try {
+        actionClass = ClassUtils.resolveClassName(classname, classLoader);
+        LOG.debug(String.format("Found external server action %s", classname));
+      } catch(IllegalArgumentException illegalArgumentException) {
+        LOG.error(String.format("Unable to find server action %s in external server action
directories", classname), illegalArgumentException);
+      }
+
+      return actionClass;
+    }
+
+    /**
+     * Load server action classes defined in Ambari source code
+     * */
+    private Class<?> getServerActionClass(String classname) throws AmbariException{
+      Class<?> actionClass = null;
+      try {
+        actionClass = Class.forName(classname);
+        if (actionClass == null) {
+          LOG.warn(String.format("Unable to load server action class: %s from Ambari", classname));
         }
       } catch (ClassNotFoundException e) {
-        throw new AmbariException("Unable to load server action class: " + classname, e);
+        LOG.error(String.format("Unable to load server action class: %s", classname), e);
       }
+      return actionClass;
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
index 119163e..7464e61 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
@@ -93,6 +93,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory
{
   protected File checksDir;
 
   /**
+   * server side action directory path
+   */
+  protected File serverActionsDir;
+
+  /**
    * service metainfo file object representation
    */
   private ServiceMetainfoXml metaInfoXml;
@@ -118,6 +123,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory
{
   protected static final String CHECKS_FOLDER_NAME = "checks";
 
   /**
+   * Server actions directory name
+   */
+  protected static final String SERVER_ACTIONS_FOLDER_NAME = "server_actions";
+
+  /**
    * service metainfo file name
    */
   private static final String SERVICE_METAINFO_FILE_NAME = "metainfo.xml";
@@ -172,6 +182,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory
{
   }
 
   /**
+   * Obtain the server side actions directory path.
+   *
+   * @return server side actions directory path
+   */
+  public File getServerActionsDir() {
+    return serverActionsDir;
+  }
+
+  /**
    * Obtain the metrics file.
    *
    * @return metrics file
@@ -303,6 +322,7 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory
{
 	  calculatePackageDirectory(stack, service);
 	  calculateUpgradesDirectory(stack, service);
 	  calculateChecksDirectory(stack, service);
+	  calculateServerActionsDirectory(stack, service);
   }
 
   /**
@@ -378,6 +398,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory
{
   }
 
   /**
+   * Sets the serverActionsDir if the dir exists and is not empty
+   * @param stack
+   * @param service
+   */
+  protected void calculateServerActionsDirectory(String stack, String service) {
+    serverActionsDir = resolveDirectory(SERVER_ACTIONS_FOLDER_NAME, stack, service);
+  }
+
+  /**
    * Unmarshal the metainfo file into its object representation.
    *
    * @throws AmbariException if the metainfo file doesn't exist or

http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
index 3b3d52c..4eb3858 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
@@ -150,6 +150,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo>
implem
     serviceInfo.setServicePackageFolder(serviceDirectory.getPackageDir());
     serviceInfo.setServiceUpgradesFolder(serviceDirectory.getUpgradesDir());
     serviceInfo.setChecksFolder(serviceDirectory.getChecksDir());
+    serviceInfo.setServerActionsFolder(serviceDirectory.getServerActionsDir());
     serviceInfo.setAdvisorFile(serviceDirectory.getAdvisorFile());
     serviceInfo.setAdvisorName(serviceDirectory.getAdvisorName(serviceInfo.getName()));
 
@@ -270,6 +271,13 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo>
implem
     }
 
     /*
+     * Use parent's server actions if the current one does not have any.
+     */
+    if (serviceInfo.getServerActionsFolder() == null) {
+      serviceInfo.setServerActionsFolder(parent.getServerActionsFolder());
+    }
+
+    /*
      * If current stack version does not specify the credential store information
      * for the service, then use parent definition.
      */

http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
index 8fe6583..f1c63bf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
@@ -305,6 +305,12 @@ public class ServiceInfo implements Validable{
   @XmlTransient
   private File checksFolder;
 
+  /**
+   * Stores the path to the server actions folder which contains server actions jars for
the given service.
+   */
+  @XmlTransient
+  private File serverActionsFolder;
+
   public boolean isDeleted() {
     return isDeleted;
   }
@@ -796,6 +802,14 @@ public class ServiceInfo implements Validable{
     this.checksFolder = checksFolder;
   }
 
+  public File getServerActionsFolder() {
+    return serverActionsFolder;
+  }
+
+  public void setServerActionsFolder(File serverActionsFolder) {
+    this.serverActionsFolder = serverActionsFolder;
+  }
+
   /**
    * Exposes (and initializes on first use) map of os-specific details.
    * @return  map of OS specific details keyed by family

http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
index dbdd043..13c32cf 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
@@ -499,6 +499,36 @@ public class ServiceModuleTest {
   }
 
   @Test
+  public void testResolve_ServerActionDirectory() throws Exception {
+    File serverActions = new File("server_actions");
+
+    // check directory specified in child only
+    ServiceInfo info = new ServiceInfo();
+    ServiceInfo parentInfo = new ServiceInfo();
+    ServiceModule child = createServiceModule(info);
+    ServiceModule parent = createServiceModule(parentInfo);
+    child.getModuleInfo().setServerActionsFolder(serverActions);
+    resolveService(child, parent);
+    assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath());
+
+    // check directory specified in parent only
+    child = createServiceModule(info);
+    parent = createServiceModule(parentInfo);
+    parent.getModuleInfo().setServerActionsFolder(serverActions);
+    resolveService(child, parent);
+    assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath());
+
+    // check directory set in both
+    info.setServerActionsFolder(serverActions);
+    child = createServiceModule(info);
+    child.getModuleInfo().setServerActionsFolder(serverActions);
+    parent = createServiceModule(parentInfo);
+    parent.getModuleInfo().setServerActionsFolder(new File("other"));
+    resolveService(child, parent);
+    assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath());
+  }
+
+  @Test
   public void testResolve_CustomCommands() throws Exception {
     List<CustomCommandDefinition> customCommands = new ArrayList<>();
     CustomCommandDefinition cmd1 = new CustomCommandDefinition();

http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
index 6617b33..bf55e6f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
@@ -140,6 +140,9 @@ public class StackManagerExtensionTest  {
     File checks = oozie.getChecksFolder();
     assertNotNull(checks);
     assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks"));
+    File serverActions = oozie.getServerActionsFolder();
+    assertNotNull(serverActions);
+    assertTrue("Server actions dir is " + serverActions.getPath(), serverActions.getPath().contains("extensions/EXT/0.1/services/OOZIE2/server_actions"));
     List<ThemeInfo> themes = oozie.getThemes();
     assertNotNull(themes);
     assertTrue("Number of themes is " + themes.size(), themes.size() == 1);
@@ -158,6 +161,9 @@ public class StackManagerExtensionTest  {
     checks = oozie.getChecksFolder();
     assertNotNull(checks);
     assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks"));
+    serverActions = oozie.getServerActionsFolder();
+    assertNotNull(serverActions);
+    assertTrue("Server actions dir is " + serverActions.getPath(), serverActions.getPath().contains("extensions/EXT/0.1/services/OOZIE2/server_actions"));
     themes = oozie.getThemes();
     assertNotNull(themes);
     assertTrue("Number of themes is " + themes.size(), themes.size() == 0);


Mime
View raw message