ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vbrodets...@apache.org
Subject ambari git commit: AMBARI-13325. Ambari-server upgrade and RU update hbase bucket cache setting incorrectly.(vbrodetskyi)
Date Tue, 13 Oct 2015 12:43:41 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1.2 afe7fa365 -> 3ed04f55c


AMBARI-13325. Ambari-server upgrade and RU update hbase bucket cache setting incorrectly.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3ed04f55
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3ed04f55
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3ed04f55

Branch: refs/heads/branch-2.1.2
Commit: 3ed04f55cb1aeacfc419e7631929e2134601f0d1
Parents: afe7fa3
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Tue Oct 13 15:41:43 2015 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Tue Oct 13 15:41:43 2015 +0300

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog210.java       | 131 ++++---------------
 .../server/upgrade/UpgradeCatalog210Test.java   |  80 +++++------
 2 files changed, 62 insertions(+), 149 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3ed04f55/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
index 8fdf640..12a1f44 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java
@@ -18,28 +18,13 @@
 
 package org.apache.ambari.server.upgrade;
 
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.regex.Matcher;
-
-import javax.persistence.EntityManager;
-import javax.persistence.Query;
-import javax.persistence.criteria.CriteriaBuilder;
-import javax.persistence.criteria.CriteriaDelete;
-import javax.persistence.criteria.Root;
-
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
@@ -60,7 +45,6 @@ import org.apache.ambari.server.orm.entities.StackEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
-import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptorContainer;
@@ -76,13 +60,26 @@ import org.eclipse.persistence.internal.databaseaccess.FieldTypeDefinition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.gson.Gson;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.persist.Transactional;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.CriteriaDelete;
+import javax.persistence.criteria.Root;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Matcher;
 
 
 /**
@@ -1507,43 +1504,6 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
     }
   }
 
-  private int getHbaseRamRecomendations(int totalMem) {
-    if (totalMem <= 4) {
-      return 1;
-    }
-    if (4 < totalMem && totalMem <= 8) {
-      return 1;
-    }
-    if (8 < totalMem && totalMem <= 16) {
-      return 2;
-    }
-    if (16 < totalMem && totalMem <= 24) {
-      return 4;
-    }
-    if (24 < totalMem && totalMem <= 48) {
-      return 8;
-    }
-    if (48 < totalMem && totalMem <= 64) {
-      return 8;
-    }
-    if (64 < totalMem && totalMem <= 72) {
-      return 8;
-    }
-    if (72 < totalMem && totalMem <= 96) {
-      return 16;
-    }
-    if (96 < totalMem && totalMem <= 128) {
-      return 24;
-    }
-    if (128 < totalMem && totalMem <= 256) {
-      return 32;
-    }
-    if (256 < totalMem) {
-      return 64;
-    }
-    return -1;
-  }
-
   protected void updateHiveConfigs() throws AmbariException {
     AmbariManagementController ambariManagementController = injector.getInstance(
             AmbariManagementController.class);
@@ -1654,43 +1614,6 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
               hbaseSiteRemoveProps.add("hbase.coprocessor.regionserver.classes");
             }
 
-            int threshold = 23;
-            int totalMem = 0;
-            String hostName = null;
-            if (cluster.getHosts("HBASE", "HBASE_MASTER").iterator().hasNext()) {
-              hostName = cluster.getHosts("HBASE", "HBASE_MASTER").iterator().next();
-            }
-            for (Host host : cluster.getHosts()) {
-              if(host.getHostName().equalsIgnoreCase(hostName)) {
-                totalMem = (int)(host.getTotalMemBytes() / (1024 * 1024));
-                break;
-              }
-            }
-
-            if (totalMem == 0) {
-              LOG.error("UpgradeCatalog210 could not retrieve total memory size from the
hosts.");
-            }
-            else {
-              if (getHbaseRamRecomendations(totalMem) > threshold) {
-                final int mb = 1024;
-                final int block_cache_heap = 8192;
-                final int regionserver_heap_size = 20480;
-                final int reserved_offheap_memory = 2048;
-                final int regionserver_total_ram = getHbaseRamRecomendations(totalMem) *
mb;
-                final int regionserver_max_direct_memory_size = regionserver_total_ram -
regionserver_heap_size;
-                final int bucketcache_offheap_memory = regionserver_max_direct_memory_size
- reserved_offheap_memory;
-
-                hbaseSiteProps.put("hbase.bucketcache.size", String.valueOf(block_cache_heap
+ bucketcache_offheap_memory));
-                hbaseSiteProps.put("hbase.bucketcache.ioengine", "offheap");
-                hbaseEnvProps.put("hbase_max_direct_memory_size", String.valueOf(regionserver_max_direct_memory_size));
-              } else {
-                hbaseSiteRemoveProps.add("hbase.bucketcache.ioengine");
-                hbaseSiteRemoveProps.add("hbase.bucketcache.size");
-                hbaseSiteRemoveProps.add("hbase.bucketcache.percentage.in.combinedcache");
-                hbaseEnvRemoveProps.add("hbase_max_direct_memory_size");
-              }
-            }
-
             updateConfigurationPropertiesForCluster(cluster, "hbase-site", hbaseSiteProps,
true, false);
             updateConfigurationPropertiesForCluster(cluster, "hbase-env", hbaseEnvProps,
true, false);
             updateConfigurationPropertiesForCluster(cluster, "hbase-site", new HashMap<String,
String>(), hbaseSiteRemoveProps, false, true);

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ed04f55/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
index e5ef3c0..482ac38 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java
@@ -18,38 +18,13 @@
 
 package org.apache.ambari.server.upgrade;
 
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertNull;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.capture;
-import static org.easymock.EasyMock.createMockBuilder;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.createStrictMock;
-import static org.easymock.EasyMock.eq;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.expectLastCall;
-import static org.easymock.EasyMock.replay;
-import static org.easymock.EasyMock.reset;
-import static org.easymock.EasyMock.verify;
-
-import java.io.File;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.net.URL;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import javax.persistence.EntityManager;
-
 import com.google.inject.AbstractModule;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.Provider;
+import com.google.inject.persist.PersistService;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.AmbariManagementController;
@@ -91,13 +66,35 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.Provider;
-import com.google.inject.persist.PersistService;
-import org.powermock.api.easymock.PowerMock;
+import javax.persistence.EntityManager;
+import java.io.File;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.createStrictMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
 
 /**
  * {@link org.apache.ambari.server.upgrade.UpgradeCatalog210} unit tests.
@@ -543,13 +540,6 @@ public class UpgradeCatalog210Test {
     expect(mockHBaseSite.getProperties()).andReturn(propertiesExpectedHBaseSite).anyTimes();
     expect(mockHBaseEnv.getProperties()).andReturn(propertiesExpectedHBaseEnv).anyTimes();
 
-    expect(mockClusterExpected.getHosts(anyObject(String.class), anyObject(String.class))).andReturn(new
HashSet<String>() {{
-      add("host_1");
-    }}).atLeastOnce();
-    expect(mockClusterExpected.getHosts()).andReturn(new HashSet<Host>(){{add(mockHost);}}).atLeastOnce();
-    expect(mockHost.getHostName()).andReturn("host_1");
-    expect(mockHost.getTotalMemBytes()).andReturn(16777216L);
-
     Capture<String> configType = new Capture<String>();
     Capture<String> configTag = new Capture<String>();
     expect(mockClusterExpected.getConfig(capture(configType), capture(configTag))).


Mime
View raw message