ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [46/50] ambari git commit: AMBARI-14689. Increase the default value for MAX_COUNTERS (aonishuk)
Date Fri, 15 Jan 2016 18:52:41 GMT
AMBARI-14689. Increase the default value for MAX_COUNTERS (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa6b943d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa6b943d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa6b943d

Branch: refs/heads/branch-dev-patch-upgrade
Commit: fa6b943de81b799ebb2d63d98f965c2515ca57ec
Parents: b9226d3
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Fri Jan 15 16:54:49 2016 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Fri Jan 15 16:54:49 2016 +0200

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog221.java       | 37 ++++++++++++
 .../stacks/HDP/2.2/upgrades/config-upgrade.xml  | 14 +++++
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml |  4 ++
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml |  6 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     |  1 +
 .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml     |  2 +
 .../2.3/services/TEZ/configuration/tez-site.xml | 12 ++++
 .../server/upgrade/UpgradeCatalog221Test.java   | 60 ++++++++++++++++++++
 8 files changed, 135 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
index 914c547..4eebb66 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java
@@ -34,6 +34,8 @@ import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.utils.VersionUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -65,6 +67,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
   private static final String SECURITY_ENABLED = "security_enabled";
   private static final String TOPOLOGY_HOST_INFO_TABLE = "topology_host_info";
   private static final String TOPOLOGY_HOST_INFO_RACK_INFO_COLUMN = "rack_info";
+  private static final String TEZ_SITE = "tez-site";
 
   @Inject
   DaoUtils daoUtils;
@@ -90,6 +93,9 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
   private static final String RANGER_KMS_JDBC_DRIVER = "ranger.ks.jpa.jdbc.driver";
   private static final String RANGER_KMS_PROPERTIES = "kms-properties";
 
+  private static final String TEZ_COUNTERS_MAX = "tez.counters.max";
+  private static final String TEZ_COUNTERS_MAX_GROUPS = "tez.counters.max.groups";
+
 
   // ----- Constructors ------------------------------------------------------
 
@@ -155,6 +161,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
     addNewConfigurationsFromXml();
     updateAlerts();
     updateOozieConfigs();
+    updateTezConfigs();
     updateRangerKmsDbksConfigs();
   }
 
@@ -363,6 +370,36 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
     }
   }
 
+  protected void updateTezConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values())
{
+      Config tezSiteProps = cluster.getDesiredConfigByType(TEZ_SITE);
+      if (tezSiteProps != null) {
+
+        // Update tez.counters.max and tez.counters.max.groups configurations
+        String tezCountersMaxProperty = tezSiteProps.getProperties().get(TEZ_COUNTERS_MAX);
+        String tezCountersMaxGroupesProperty = tezSiteProps.getProperties().get(TEZ_COUNTERS_MAX_GROUPS);
+
+        StackId stackId = cluster.getCurrentStackVersion();
+        boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP")
&&
+            VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0);
+
+        if (isStackNotLess23) {
+          Map<String, String> updates = new HashMap<String, String>();
+          if (tezCountersMaxProperty != null && tezCountersMaxProperty.equals("2000"))
{
+            updates.put(TEZ_COUNTERS_MAX, "10000");
+          }
+          if (tezCountersMaxGroupesProperty != null && tezCountersMaxGroupesProperty.equals("1000"))
{
+            updates.put(TEZ_COUNTERS_MAX_GROUPS, "3000");
+          }
+          if (!updates.isEmpty()) {
+            updateConfigurationPropertiesForCluster(cluster, TEZ_SITE, updates, true, false);
+          }
+        }
+      }
+    }
+  }
+
   protected void updateRangerKmsDbksConfigs() throws AmbariException {
     AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
index c2844fa..ffc7ffb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/config-upgrade.xml
@@ -107,6 +107,20 @@
             <type>tez-site</type>
             <set key="tez.lib.uris" value="/hdp/apps/${hdp.version}/tez/tez.tar.gz"/>
           </definition>
+
+          <definition xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties">
+            <condition type="tez-site" key="tez.counters.max" value="2000">
+              <type>tez-site</type>
+              <key>tez.counters.max</key>
+              <value>10000</value>
+            </condition>
+            <condition type="tez-site" key="tez.counters.max.groups" value="1000">
+              <type>tez-site</type>
+              <key>tez.counters.max.groups</key>
+              <value>3000</value>
+            </condition>
+          </definition>
+
         </changes>
       </component>
     </service>

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
index 5dfa2bd..816fe13 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
@@ -412,6 +412,10 @@
       </execute-stage>
 
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes
for Tez">
+        <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/>
+      </execute-stage>
+
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes
for Tez">
         <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_tez_lib_uris_property"/>
       </execute-stage>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index 00b9e96..5166c4b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -446,6 +446,10 @@
       </execute-stage>
 
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes
for Tez">
+        <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/>
+      </execute-stage>
+
+      <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes
for Tez">
         <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
       </execute-stage>
 
@@ -1358,4 +1362,4 @@
 
     </service>
   </processing>
-</upgrade>
\ No newline at end of file
+</upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index fd64c8e..375da12 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -626,6 +626,7 @@
       <component name="TEZ_CLIENT">
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/>
+          <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/>
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_tez_lib_uris_property"/>
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_keep_ats_v1"/>
         </pre-upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index 7637830..ea3d6a6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -634,6 +634,8 @@
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_3_0_0_tez_client_adjust_properties"/>
 
+          <task xsi:type="configure" id="hdp_2_2_0_0_tez_client_adjust_tez_counters_properties"/>
+
           <!-- Apply HDP 2.4 since same as that in HDP 2.3 -->
           <task xsi:type="configure" id="hdp_2_4_0_0_tez_client_adjust_tez_lib_uris_property"/>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml
index a80da09..b2db250 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/TEZ/configuration/tez-site.xml
@@ -103,4 +103,16 @@
     <value>org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService</value>
   </property>
 
+  <property>
+    <name>tez.counters.max</name>
+    <value>10000</value>
+    <description>The number of allowed counters for the executing DAG</description>
+  </property>
+
+  <property>
+    <name>tez.counters.max.groups</name>
+    <value>3000</value>
+    <description>The number of allowed counter groups for the executing DAG</description>
+  </property>
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa6b943d/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
index c14149f..5a14bc9 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java
@@ -44,6 +44,9 @@ import org.apache.ambari.server.orm.entities.StackEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.RepositoryInfo;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.StackInfo;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.easymock.Capture;
 import org.easymock.EasyMock;
@@ -55,6 +58,7 @@ import org.junit.Test;
 import javax.persistence.EntityManager;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -138,12 +142,14 @@ public class UpgradeCatalog221Test {
     Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml");
     Method updateAlerts = UpgradeCatalog221.class.getDeclaredMethod("updateAlerts");
     Method updateOozieConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateOozieConfigs");
+    Method updateTezConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateTezConfigs");
     Method updateRangerKmsDbksConfigs = UpgradeCatalog221.class.getDeclaredMethod("updateRangerKmsDbksConfigs");
 
     UpgradeCatalog221 upgradeCatalog221 = createMockBuilder(UpgradeCatalog221.class)
       .addMockedMethod(addNewConfigurationsFromXml)
       .addMockedMethod(updateAlerts)
       .addMockedMethod(updateOozieConfigs)
+      .addMockedMethod(updateTezConfigs)
       .addMockedMethod(updateRangerKmsDbksConfigs)
       .createMock();
 
@@ -153,6 +159,8 @@ public class UpgradeCatalog221Test {
     expectLastCall().once();
     upgradeCatalog221.updateOozieConfigs();
     expectLastCall().once();
+    upgradeCatalog221.updateTezConfigs();
+    expectLastCall().once();
     upgradeCatalog221.updateRangerKmsDbksConfigs();
     expectLastCall().once();
 
@@ -246,6 +254,58 @@ public class UpgradeCatalog221Test {
   }
 
   @Test
+  public void testUpdateTezConfigs() throws Exception {
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+    final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
+    final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
+    final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+
+    final Config tezSiteConf = easyMockSupport.createNiceMock(Config.class);
+    final Map<String, String> propertiesTezSite = new HashMap<String, String>()
{{
+      put("tez.counters.max", "2000");
+      put("tez.counters.max.groups", "1000");
+    }};
+
+    StackId stackId = new StackId("HDP","2.3");
+
+    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+      @Override
+      protected void configure() {
+        bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
+        bind(Clusters.class).toInstance(mockClusters);
+        bind(EntityManager.class).toInstance(entityManager);
+
+        bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+        bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+      }
+    });
+
+    expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once();
+    expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+      put("normal", mockClusterExpected);
+    }}).atLeastOnce();
+
+
+    expect(mockClusterExpected.getDesiredConfigByType("tez-site")).andReturn(tezSiteConf).atLeastOnce();
+    expect(tezSiteConf.getProperties()).andReturn(propertiesTezSite).once();
+    expect(mockClusterExpected.getCurrentStackVersion()).andReturn(stackId).once();
+
+    UpgradeCatalog221 upgradeCatalog221 = createMockBuilder(UpgradeCatalog221.class)
+        .withConstructor(Injector.class)
+        .withArgs(mockInjector)
+        .addMockedMethod("updateConfigurationPropertiesForCluster", Cluster.class, String.class,
+            Map.class, boolean.class, boolean.class)
+        .createMock();
+    Map<String, String> updates = new HashMap<String, String>();
+    updates.put("tez.counters.max", "10000");
+    updates.put("tez.counters.max.groups", "3000");
+    upgradeCatalog221.updateConfigurationPropertiesForCluster(mockClusterExpected, "tez-site",
+        updates, true, false);
+    expectLastCall().once();
+
+  }
+
+  @Test
   public void testUpdateRangerKmsDbksConfigs() throws Exception {
     EasyMockSupport easyMockSupport = new EasyMockSupport();
     final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);


Mime
View raw message