ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [2/2] ambari git commit: AMBARI-18062. Review the implementation of the Falcon/Atlas hook enabling/disabling (aonishuk)
Date Mon, 08 Aug 2016 15:36:50 GMT
AMBARI-18062. Review the implementation of the Falcon/Atlas hook enabling/disabling (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/03536e16
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/03536e16
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/03536e16

Branch: refs/heads/branch-2.4
Commit: 03536e1699f15fa654d4104972b27fc8072c9e63
Parents: 8444050
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Mon Aug 8 18:36:39 2016 +0300
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Mon Aug 8 18:36:39 2016 +0300

----------------------------------------------------------------------
 .../libraries/functions/setup_atlas_hook.py      | 11 +++++------
 .../ambari/server/upgrade/UpgradeCatalog240.java | 19 ++++++++++++++++++-
 .../0.5.0.2.1/configuration/falcon-env.xml       |  2 +-
 .../FALCON/0.5.0.2.1/package/scripts/falcon.py   |  3 ++-
 .../0.5.0.2.1/package/scripts/params_linux.py    |  2 +-
 .../server/upgrade/UpgradeCatalog240Test.java    | 15 +++++++++++++++
 6 files changed, 42 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
index 26a0dd6..2fdcf9d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/setup_atlas_hook.py
@@ -173,9 +173,8 @@ def setup_atlas_jar_symlinks(hook_name, jar_source_dir):
         if os.path.isfile(atlas_hook_file_name):
           Link(source_lib_file_name, to=atlas_hook_file_name)
 
-def install_atlas_hook_packages():
-  import params
-
-  if not params.host_sys_prepped:
-    Package(params.atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else params.atlas_plugin_package,
-            retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count)
\ No newline at end of file
+def install_atlas_hook_packages(atlas_plugin_package, atlas_ubuntu_plugin_package, host_sys_prepped,
+                                agent_stack_retry_on_unavailability, agent_stack_retry_count):
+  if not host_sys_prepped:
+    Package(atlas_ubuntu_plugin_package if OSCheck.is_ubuntu_family() else atlas_plugin_package,
+            retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
index 8c22eeb..b3613af 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java
@@ -2201,6 +2201,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
   /**
    * Updates the Falcon-related configurations for the clusters managed by this Ambari
    * Removes falcon_store_uri from falcon-env.
+   * Appends Atlas hook classpath to FALCON_EXTRA_CLASS_PATH from falcon-env if it doesn't
contain it
    * Appends '{{atlas_application_class_addition}}' to *.application.services from falcon-startup.properties
if it doesn't contain it.
    *
    * @throws AmbariException if an error occurs while updating the configurations
@@ -2211,14 +2212,30 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog {
     Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
 
     for (final Cluster cluster : clusterMap.values()) {
-      // Remove falcon_store_uri from falcon-env.
       Config falconEnvConfig = cluster.getDesiredConfigByType("falcon-env");
       if (falconEnvConfig != null) {
+        // Remove falcon_store_uri from falcon-env.
         Map<String, String> falconEnvEnvProperties = falconEnvConfig.getProperties();
         if (falconEnvEnvProperties.containsKey("falcon_store_uri")) {
           LOG.info("Removing property falcon_store_uri from falcon-env");
           removeConfigurationPropertiesFromCluster(cluster, "falcon-env", Collections.singleton("falcon_store_uri"));
         }
+
+        // Append Atlas hook classpath to FALCON_EXTRA_CLASS_PATH from falcon-env if it doesn't
contain it
+        // This is necessary for stack upgrades to 2.5 after the Ambari upgrade
+        final String envContentPropertyName = "content";
+        String contentValue = falconEnvConfig.getProperties().get(envContentPropertyName);
+        if (contentValue != null) {
+          final String atlasHookCPAddition = "{{atlas_hook_cp}}";
+          if (!contentValue.contains(atlasHookCPAddition)) {
+            LOG.info("Appending '{}' to FALCON_EXTRA_CLASS_PATH from falcon-env since it
doesn't contain it", atlasHookCPAddition);
+            String newValue = contentValue + "\n\n{% if falcon_atlas_support %}\n" +
+                "# Add the Atlas Falcon hook to the Falcon classpath\n" +
+                "export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH}\n"
+
+                "{% endif %}";
+            updateConfigurationPropertiesForCluster(cluster, "falcon-env", Collections.singletonMap(envContentPropertyName,
newValue), null, true, false);
+          }
+        }
       }
 
       // Update falcon-startup.properties/*.application.services.

http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
index 48984fd..09cced6 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml
@@ -162,8 +162,8 @@ export FALCON_DATA_DIR={{falcon_embeddedmq_data}}
 # Where do you want to expand the war file. By Default it is in /server/webapp dir under
the base install dir.
 #export FALCON_EXPANDED_WEBAPP_DIR=
 
-# Add the Atlas Falcon hook to the Falcon classpath
 {% if falcon_atlas_support %}
+# Add the Atlas Falcon hook to the Falcon classpath
 export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH}
 {% endif %}
     </value>

http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index 8025e56..7a5cecb 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -120,7 +120,8 @@ def falcon(type, action = None, upgrade_type=None):
     # Generate atlas-application.properties.xml file
     if params.falcon_atlas_support:
       # If Atlas is added later than Falcon, this package will be absent.
-      install_atlas_hook_packages()
+      install_atlas_hook_packages(params.atlas_plugin_package, params.atlas_ubuntu_plugin_package,
params.host_sys_prepped,
+                                  params.agent_stack_retry_on_unavailability, params.agent_stack_retry_count)
 
       atlas_hook_filepath = os.path.join(params.falcon_conf_dir, params.atlas_hook_filename)
       setup_atlas_hook(SERVICE.FALCON, params.falcon_atlas_application_properties, atlas_hook_filepath,
params.falcon_user, params.user_group)

http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 2220480..a3a4e54 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -166,7 +166,7 @@ if has_atlas_in_cluster():
       atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ
else format('{stack_root}/current/atlas-server')
       atlas_hook_cp = atlas_conf_dir + os.pathsep + os.path.join(atlas_home_dir, "hook",
"falcon", "*") + os.pathsep
     elif check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, stack_version_formatted):
-      atlas_hook_cp = format('{stack_root}/current/atlas-client/hook/falcon/*')
+      atlas_hook_cp = format('{stack_root}/current/atlas-client/hook/falcon/*') + os.pathsep
 
 atlas_application_class_addition = ""
 if falcon_atlas_support:

http://git-wip-us.apache.org/repos/asf/ambari/blob/03536e16/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
index 3b42984..734cc7c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java
@@ -779,6 +779,7 @@ public class UpgradeCatalog240Test {
 
     final Map<String, String> propertiesExpectedFalconEnv = new HashMap<String,
String>();
     propertiesExpectedFalconEnv.put("falcon_store_uri", "file:///hadoop/falcon/store");
+    propertiesExpectedFalconEnv.put("content", "# content");
     propertiesExpectedFalconEnv.put("property", "value");
 
     final String applicationServicesOldPropertyValue =
@@ -834,6 +835,10 @@ public class UpgradeCatalog240Test {
     expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-env"),
         capture(falconCapture), anyString(), (Map<String, Map<String, String>>)
anyObject())).andReturn(null).once();
 
+    Capture<Map<String, String>> falconCapture2 =  newCapture();
+    expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-env"),
+        capture(falconCapture2), anyString(), (Map<String, Map<String, String>>)
anyObject())).andReturn(null).once();
+
     Capture<Map<String, String>> falconStartupCapture =  newCapture();
     expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-startup.properties"),
         capture(falconStartupCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once();
@@ -842,9 +847,19 @@ public class UpgradeCatalog240Test {
     mockInjector.getInstance(UpgradeCatalog240.class).updateFalconConfigs();
     easyMockSupport.verifyAll();
 
+    final String expectredEnvContent = "# content\n" +
+                                       "\n" +
+                                       "{% if falcon_atlas_support %}\n" +
+                                       "# Add the Atlas Falcon hook to the Falcon classpath\n"
+
+                                       "export FALCON_EXTRA_CLASS_PATH={{atlas_hook_cp}}${FALCON_EXTRA_CLASS_PATH}\n"
+
+                                       "{% endif %}";
+
     assertEquals("value", falconCapture.getValue().get("property"));
+    assertEquals("# content", falconCapture.getValue().get("content"));
     assertNull(falconCapture.getValue().get("falcon_store_uri"));
 
+    assertEquals(expectredEnvContent, falconCapture2.getValue().get("content"));
+
     assertEquals("value", falconStartupCapture.getValue().get("property"));
     assertEquals(applicationServicesExpectedPropertyValue, falconStartupCapture.getValue().get("*.application.services"));
   }


Mime
View raw message