ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-11767 Configs: hive warning when using the default values during install (dsen)
Date Sun, 07 Jun 2015 14:12:03 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 b1ee902bc -> b398f7060


AMBARI-11767 Configs: hive warning when using the default values during install (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b398f706
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b398f706
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b398f706

Branch: refs/heads/branch-2.1
Commit: b398f70601e6da8b25ebc63d1f21ebefe192e8fa
Parents: b1ee902
Author: Dmytro Sen <dsen@apache.org>
Authored: Sun Jun 7 17:07:08 2015 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Sun Jun 7 17:08:17 2015 +0300

----------------------------------------------------------------------
 .../common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml | 2 +-
 .../stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml    | 2 +-
 .../src/main/resources/stacks/HDP/2.1/services/stack_advisor.py | 3 ++-
 .../stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml    | 2 +-
 .../src/main/resources/stacks/HDP/2.2/services/stack_advisor.py | 2 +-
 .../src/test/python/stacks/2.1/common/test_stack_advisor.py     | 5 +++--
 6 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b398f706/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index 82eafde..f4e4b25 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -255,7 +255,7 @@ limitations under the License.
     <value>2147483648</value>
     <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
       is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
-      converted to a mapjoin(there is no conditional task). The default is 10MB.
+      converted to a mapjoin(there is no conditional task).
     </description>
     <display-name>For Map Join, per Map memory threshold</display-name>
     <value-attributes>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b398f706/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
index f0b5a04..182acfe 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml
@@ -253,7 +253,7 @@ limitations under the License.
     <value>1000000000</value>
     <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
       is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
-      converted to a mapjoin(there is no conditional task). The default is 10MB.
+      converted to a mapjoin(there is no conditional task).
     </description>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b398f706/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
index 2c80263..44595be 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py
@@ -40,8 +40,9 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
   def recommendHiveConfigurations(self, configurations, clusterData, services, hosts):
     containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
     containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
+    container_size_bytes = int(containerSize)*1024*1024
     putHiveProperty = self.putProperty(configurations, "hive-site", services)
-    putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(round(containerSize
/ 3)) * 1048576)
+    putHiveProperty('hive.auto.convert.join.noconditionaltask.size', int(round(container_size_bytes
/ 3)))
     putHiveProperty('hive.tez.java.opts', "-server -Xmx" + str(int(round((0.8 * containerSize)
+ 0.5)))
                     + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC
-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps")
     putHiveProperty('hive.tez.container.size', containerSize)

http://git-wip-us.apache.org/repos/asf/ambari/blob/b398f706/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
index 5e5c64f..5f24e7c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml
@@ -1407,7 +1407,7 @@ limitations under the License.
     <value>52428800</value>
     <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
       is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
-      converted to a mapjoin(there is no conditional task). The default is 10MB.
+      converted to a mapjoin(there is no conditional task).
     </description>
     <display-name>For Map Join, per Map memory threshold</display-name>
     <value-attributes>

http://git-wip-us.apache.org/repos/asf/ambari/blob/b398f706/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 5839149..36a6b0b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -330,7 +330,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     container_size = configurations["hive-site"]["properties"]["hive.tez.container.size"]
     container_size_bytes = int(container_size)*1024*1024
     # Memory
-    putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(container_size_bytes/3))
+    putHiveSiteProperty("hive.auto.convert.join.noconditionaltask.size", int(round(container_size_bytes/3)))
     putHiveSitePropertyAttribute("hive.auto.convert.join.noconditionaltask.size", "maximum",
container_size_bytes)
     putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/b398f706/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
index 9473785..3728d8e 100644
--- a/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py
@@ -79,7 +79,7 @@ class TestHDP21StackAdvisor(TestCase):
     expected = {
       "hive-site": {
         "properties": {
-          "hive.auto.convert.join.noconditionaltask.size": "718274560",
+          "hive.auto.convert.join.noconditionaltask.size": "718624085",
           "hive.tez.java.opts": "-server -Xmx1645m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8
-XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
           "hive.tez.container.size": "2056"
         }
@@ -87,6 +87,7 @@ class TestHDP21StackAdvisor(TestCase):
     }
 
     self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, None, None)
+    self.maxDiff = None
     self.assertEquals(configurations, expected)
 
   def test_recommendHiveConfigurations_mapMemoryMoreThan2048(self):
@@ -272,4 +273,4 @@ class TestHDP21StackAdvisor(TestCase):
                     ]
 
     res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations,
'', '')
-    self.assertEquals(res, res_expected)
\ No newline at end of file
+    self.assertEquals(res, res_expected)


Mime
View raw message