ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jai...@apache.org
Subject [17/19] ambari git commit: AMBARI-17798. RU/EU: Add falcon to oozie admin user for HDP 2.5 (Venkat Ranganathan via alejandro)
Date Thu, 21 Jul 2016 21:02:32 GMT
AMBARI-17798. RU/EU: Add falcon to oozie admin user for HDP 2.5 (Venkat Ranganathan via alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18ba883b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18ba883b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18ba883b

Branch: refs/heads/branch-embedded-views
Commit: 18ba883b314f2b8259a648b4ab1fd36f4a5cfda0
Parents: aff87ed
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Thu Jul 21 13:44:47 2016 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Thu Jul 21 13:44:47 2016 -0700

----------------------------------------------------------------------
 .../upgrades/FixOozieAdminUsers.java            |  95 ++++++++++++
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml |   1 +
 .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml     |   2 +
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml |   4 +-
 .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml     |   2 +
 .../stacks/HDP/2.5/services/stack_advisor.py    |  47 ++++--
 .../upgrades/FixOozieAdminUsersTest.java        | 154 +++++++++++++++++++
 7 files changed, 287 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java
new file mode 100644
index 0000000..3a06476
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsers.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Inject;
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * During stack upgrade, update lzo codec path in mapreduce.application.classpath and
+ * at tez.cluster.additional.classpath.prefix to look like
+ * /usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar
+ */
+public class FixOozieAdminUsers extends AbstractServerAction {
+  private static final String TARGET_OOZIE_CONFIG_TYPE = "oozie-env";
+  private static final String OOZIE_ADMIN_USERS_PROP = "oozie_admin_users";
+  private static final String FALCON_CONFIG_TYPE = "falcon-env";
+  private static final String FALCON_USER_PROP = "falcon_user";
+
+
+  @Inject
+  private Clusters clusters;
+
+  @Override
+  public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+    throws AmbariException, InterruptedException {
+    String clusterName = getExecutionCommand().getClusterName();
+    Cluster cluster = clusters.getCluster(clusterName);
+    Config oozieConfig = cluster.getDesiredConfigByType(TARGET_OOZIE_CONFIG_TYPE);
+    Config falconConfig = cluster.getDesiredConfigByType(FALCON_CONFIG_TYPE);
+
+    if (falconConfig == null) {
+      return  createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+              "Falcon configuration unavailable", "");
+
+    }
+    if (oozieConfig == null) {
+      return  createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+              "Oozie configuration unavailable", "");
+    }
+
+    Map<String, String> oozieProperties = oozieConfig.getProperties();
+    String currentOozieAdmins = oozieProperties.get(OOZIE_ADMIN_USERS_PROP);
+    if (currentOozieAdmins.isEmpty()) {
+      currentOozieAdmins = "";
+    }
+    Map<String, String> falconProperties = falconConfig.getProperties();
+    String falconUser = falconProperties.get(FALCON_USER_PROP);
+    if (StringUtils.isBlank(falconUser)) {
+      return  createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+              "Falcon user not set", "");
+    }
+
+    if (currentOozieAdmins.indexOf(falconUser) >= 0) {
+      return  createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+              "Falcon user already member of Oozie admins", "");
+    }
+
+    String newOozieAdminUsers = currentOozieAdmins + "," + falconUser;
+
+    oozieProperties.put(OOZIE_ADMIN_USERS_PROP, newOozieAdminUsers);
+
+    oozieConfig.setProperties(oozieProperties);
+    oozieConfig.persist(false);
+
+    return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+            String.format("Set oozie admin users to %s", newOozieAdminUsers), "");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 7be5a19..a61b599 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -147,6 +147,7 @@
         <task xsi:type="manual">
           <message>Before continuing, please backup the Oozie Server database referenced
by the Oozie server located on {{hosts.all}}.</message>
         </task>
+        <task xsi:type="server_action" summary="Fix oozie admin users" class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
       </execute-stage>
 
       <execute-stage service="HIVE" component="HIVE_METASTORE" title="Backup Hive Metastore">

http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 7f277fb..4d5fd07 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -883,6 +883,8 @@
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
+          <task xsi:type="server_action" summary="Fixing Oozie admin users"
+                class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
         </pre-upgrade>
 
         <pre-downgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 2daa9b8..065a933 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -287,9 +287,11 @@
       <!--OOZIE-->
       <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes
for Oozie Server">
         <task xsi:type="server_action" summary="Adjusting Oozie properties" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation"/>
+        <task xsi:type="server_action" summary="Fix oozie admin users" class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
       </execute-stage>
 
-      <!--FALCON-->
+
+    <!--FALCON-->
       <execute-stage service="FALCON" component="FALCON_SERVER" title="Apply config changes
for Falcon">
         <task xsi:type="configure" id="hdp_2_5_0_0_falcon_server_adjust_services_property"/>
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 43c6265..d152f34 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -862,6 +862,8 @@
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
+          <task xsi:type="server_action" summary="Fixing Oozie admin users"
+                class="org.apache.ambari.server.serveraction.upgrades.FixOozieAdminUsers"/>
         </pre-upgrade>
 
         <pre-downgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index a7eed00..be2a011 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -36,36 +36,49 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
   def recommendOozieConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor,self).recommendOozieConfigurations(configurations, clusterData,
services, hosts)
     putOozieEnvProperty = self.putProperty(configurations, "oozie-env", services)
-    if not "FALCON_SERVER" in clusterData["components"] :
-      print "Falcon is not part of the installation"
-      return
 
     if not "oozie-env" in services["configurations"] :
-      print "No oozie env in configurations"
+      Logger.info("No oozie configurations available")
       return
 
-    if "falcon-env" in services["configurations"] and "falcon_user" \
-        in services["configurations"]["falcon-env"]["properties"] :
-      falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"]
-      print "Falcon user from configuration " + falconUser
-    else :
-      falconUser = 'falcon'
-      print "Defaulting falcon user to " + falconUser
+    falconUser = 'falcon'
+
+    if "falcon-env" in services["configurations"] :
+      if "falcon_user" in services["configurations"]["falcon-env"]["properties"] :
+        falconUser = services["configurations"]["falcon-env"]["properties"]["falcon_user"]
+        Logger.info("Falcon user from configuration: %s " % falconUser)
+
+    Logger.info("Falcon user : %s" % falconUser)
+
+    oozieUser = 'oozie'
 
     if "oozie_user" \
       in services["configurations"]["oozie-env"]["properties"] :
       oozieUser = services["configurations"]["oozie-env"]["properties"]["oozie_user"]
-      print "Oozie user from configuration " + oozieUser
+      Logger.info("Oozie user from configuration %s" % oozieUser)
+
+    Logger.info("Oozie user %s" % oozieUser)
+
+    if "oozie_admin_users" \
+            in services["configurations"]["oozie-env"]["properties"] :
+      currentAdminUsers =  services["configurations"]["oozie-env"]["properties"]["oozie_admin_users"]
+      Logger.info("Oozie admin users from configuration %s" % currentAdminUsers)
     else :
-      oozieUser = 'oozie'
-      print "Defaulting oozie user to " + oozieUser
+      currentAdminUsers = "{0}, oozie-admin".format(oozieUser)
+      Logger.info("Setting default oozie admin users to %s" % currentAdminUsers)
+
+
+    if falconUser in currentAdminUsers :
+      Logger.info("Falcon user %s already member of  oozie admin users " % falconUser)
+      return
+
+    newAdminUsers = "{0},{1}".format(currentAdminUsers, falconUser)
+
+    Logger.info("new oozie admin users : %s" % newAdminUsers)
 
-    newAdminUsers = "{0},oozie-admin,{1}".format(oozieUser, falconUser)
-    print "Setting new  oozie admin user to " + newAdminUsers
     services["forced-configurations"].append({"type" : "oozie-env", "name" : "oozie_admin_users"})
     putOozieEnvProperty("oozie_admin_users", newAdminUsers)
 
-
   def createComponentLayoutRecommendations(self, services, hosts):
     parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
       services, hosts)

http://git-wip-us.apache.org/repos/asf/ambari/blob/18ba883b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java
new file mode 100644
index 0000000..314e955
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixOozieAdminUsersTest.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import com.google.inject.Injector;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigImpl;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests OozieConfigCalculation logic
+ */
+public class FixOozieAdminUsersTest {
+  private Injector injector;
+  private Clusters clusters;
+  private Cluster cluster;
+  private Field clustersField;
+
+  @Before
+  public void setup() throws Exception {
+    injector = EasyMock.createMock(Injector.class);
+    clusters = EasyMock.createMock(Clusters.class);
+    cluster = EasyMock.createMock(Cluster.class);
+
+
+    Config falconEnvConfig = new ConfigImpl("falcon-env") {
+      Map<String, String> mockProperties = new HashMap<String, String>() {{
+        put("falcon_user", "falcon");
+      }};
+
+      @Override
+      public Map<String, String> getProperties() {
+        return mockProperties;
+      }
+
+      @Override
+      public void setProperties(Map<String, String> properties) {
+        mockProperties.putAll(properties);
+      }
+
+      @Override
+      public void persist(boolean newConfig) {
+        // no-op
+      }
+    };
+    Config oozieEnvConfig = new ConfigImpl("oozie-env") {
+      Map<String, String> mockProperties = new HashMap<String, String>() {{
+        put("oozie_admin_users", "oozie, oozie-admin");
+      }};
+      @Override
+      public Map<String, String> getProperties() {
+        return mockProperties;
+      }
+
+      @Override
+      public void setProperties(Map<String, String> properties) {
+        mockProperties.putAll(properties);
+      }
+
+      @Override
+      public void persist(boolean newConfig) {
+        // no-op
+      }
+    };
+    expect(cluster.getDesiredConfigByType("falcon-env")).andReturn(falconEnvConfig).atLeastOnce();
+    expect(cluster.getDesiredConfigByType("oozie-env")).andReturn(oozieEnvConfig).atLeastOnce();
+
+    expect(clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+    expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce();
+    replay(injector, clusters);
+
+    clustersField = FixOozieAdminUsers.class.getDeclaredField("clusters");
+    clustersField.setAccessible(true);
+
+  }
+
+  /**
+   * Makes sure falcon user is added to oozie admin users
+   * @throws Exception
+   */
+  @Test
+  public void testOozieAdminUserUpdated() throws Exception {
+
+    Map<String, String> commandParams = new HashMap<String, String>();
+    commandParams.put("clusterName", "c1");
+
+    ExecutionCommand executionCommand = new ExecutionCommand();
+    executionCommand.setCommandParams(commandParams);
+    executionCommand.setClusterName("c1");
+
+    HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+    expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+    expect(hrc.getStageId()).andReturn(2L).anyTimes();
+    expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+    replay(cluster, hrc);
+
+    FixOozieAdminUsers action = new FixOozieAdminUsers();
+    clustersField.set(action, clusters);
+
+    action.setExecutionCommand(executionCommand);
+    action.setHostRoleCommand(hrc);
+
+    CommandReport report = action.execute(null);
+    assertNotNull(report);
+
+
+    Cluster c = clusters.getCluster("c1");
+    Config oozieConfig = c.getDesiredConfigByType("oozie-env");
+    Config falconConfig = c.getDesiredConfigByType("falcon-env");
+
+    Map<String, String> oozieConfigMap = oozieConfig.getProperties();
+    Map<String, String> falconConfigMap = falconConfig.getProperties();
+
+    assertTrue(oozieConfigMap.containsKey("oozie_admin_users"));
+    assertTrue(falconConfigMap.containsKey("falcon_user"));
+    String oozieAdminUsers = oozieConfigMap.get("oozie_admin_users");
+    String falconUser = falconConfigMap.get("falcon_user");
+
+    assertTrue(oozieAdminUsers.indexOf(falconUser) != -1);
+
+  }
+
+}


Mime
View raw message