incubator-ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject svn commit: r1482557 - in /incubator/ambari/trunk: CHANGES.txt ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml
Date Tue, 14 May 2013 20:21:34 GMT
Author: swagle
Date: Tue May 14 20:21:34 2013
New Revision: 1482557

URL: http://svn.apache.org/r1482557
Log:
AMBARI-2116. Add parameters to improve HIVE performance. With descriptions. (swagle)

Modified:
    incubator/ambari/trunk/CHANGES.txt
    incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml
    incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml

Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1482557&r1=1482556&r2=1482557&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Tue May 14 20:21:34 2013
@@ -850,6 +850,9 @@ Trunk (unreleased changes):
 
  BUG FIXES
 
+ AMBARI-2116. Add parameters to improve HIVE performance. With decriptions. 
+ (swagle)
+
  AMBARI-2127. Service check should not emit log with Exceptions considering
  its a positive scenario. (smohanty)
 

Modified: incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml?rev=1482557&r1=1482556&r2=1482557&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml
(original)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDP/1.3.0/services/HIVE/configuration/hive-site.xml
Tue May 14 20:21:34 2013
@@ -143,16 +143,19 @@ limitations under the License.
   <property>
     <name>hive.enforce.bucketing</name>
     <value>true</value>
+    <description>Whether bucketing is enforced. If true, while inserting into the table,
bucketing is enforced.</description>
   </property>
 
   <property>
     <name>hive.enforce.sorting</name>
     <value>true</value>
+    <description>Whether sorting is enforced. If true, while inserting into the table,
sorting is enforced.</description>
   </property>
 
   <property>
     <name>hive.map.aggr</name>
     <value>true</value>
+    <description>Whether to use map-side aggregation in Hive Group By queries.</description>
   </property>
 
   <property>
@@ -168,16 +171,22 @@ limitations under the License.
   <property>
     <name>hive.mapred.reduce.tasks.speculative.execution</name>
     <value>false</value>
+    <description>Whether speculative execution for reducers should be turned on.</description>
   </property>
 
   <property>
     <name>hive.auto.convert.join</name>
     <value>true</value>
+    <description>Whether Hive enable the optimization about converting common
+      join into mapjoin based on the input file size.</description>
   </property>
 
   <property>
     <name>hive.auto.convert.sortmerge.join</name>
     <value>true</value>
+    <description>Will the join be automatically converted to a sort-merge join, if
the joined tables pass
+      the criteria for sort-merge join.
+    </description>
   </property>
 
   <property>
@@ -188,21 +197,38 @@ limitations under the License.
   <property>
     <name>hive.auto.convert.join.noconditionaltask</name>
     <value>true</value>
+    <description>Whether Hive enable the optimization about converting common join
into mapjoin based on the input file
+      size. If this paramater is on, and the sum of size for n-1 of the tables/partitions
for a n-way join is smaller than the
+      specified size, the join is directly converted to a mapjoin (there is no conditional
task).
+    </description>
   </property>
 
   <property>
     <name>hive.auto.convert.join.noconditionaltask.size</name>
     <value>1000000000</value>
+    <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
+      is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
+      converted to a mapjoin(there is no conditional task). The default is 10MB.
+    </description>
   </property>
 
   <property>
     <name>hive.optimize.reducededuplication.min.reducer</name>
     <value>1</value>
+    <description>Reduce deduplication merges two RSs by moving key/parts/reducer-num
of the child RS to parent RS.
+      That means if reducer-num of the child RS is fixed (order by or forced bucketing) and
small, it can make very slow, single MR.
+      The optimization will be disabled if number of reducers is less than specified value.
+    </description>
   </property>
 
   <property>
     <name>hive.optimize.mapjoin.mapreduce</name>
     <value>true</value>
+    <description>If hive.auto.convert.join is off, this parameter does not take
+      affect. If it is on, and if there are map-join jobs followed by a map-reduce
+      job (for e.g a group by), each map-only job is merged with the following
+      map-reduce job.
+    </description>
   </property>
 
 </configuration>

Modified: incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml?rev=1482557&r1=1482556&r2=1482557&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml
(original)
+++ incubator/ambari/trunk/ambari-server/src/main/resources/stacks/HDPLocal/1.3.0/services/HIVE/configuration/hive-site.xml
Tue May 14 20:21:34 2013
@@ -143,16 +143,19 @@ limitations under the License.
   <property>
     <name>hive.enforce.bucketing</name>
     <value>true</value>
+    <description>Whether bucketing is enforced. If true, while inserting into the table,
bucketing is enforced.</description>
   </property>
 
   <property>
     <name>hive.enforce.sorting</name>
     <value>true</value>
+    <description>Whether sorting is enforced. If true, while inserting into the table,
sorting is enforced.</description>
   </property>
 
   <property>
     <name>hive.map.aggr</name>
     <value>true</value>
+    <description>Whether to use map-side aggregation in Hive Group By queries.</description>
   </property>
 
   <property>
@@ -168,16 +171,22 @@ limitations under the License.
   <property>
     <name>hive.mapred.reduce.tasks.speculative.execution</name>
     <value>false</value>
+    <description>Whether speculative execution for reducers should be turned on.</description>
   </property>
 
   <property>
     <name>hive.auto.convert.join</name>
     <value>true</value>
+    <description>Whether Hive enable the optimization about converting common
+      join into mapjoin based on the input file size.</description>
   </property>
 
   <property>
     <name>hive.auto.convert.sortmerge.join</name>
     <value>true</value>
+    <description>Will the join be automatically converted to a sort-merge join, if
the joined tables pass
+      the criteria for sort-merge join.
+    </description>
   </property>
 
   <property>
@@ -188,21 +197,38 @@ limitations under the License.
   <property>
     <name>hive.auto.convert.join.noconditionaltask</name>
     <value>true</value>
+    <description>Whether Hive enable the optimization about converting common join
into mapjoin based on the input file
+      size. If this paramater is on, and the sum of size for n-1 of the tables/partitions
for a n-way join is smaller than the
+      specified size, the join is directly converted to a mapjoin (there is no conditional
task).
+    </description>
   </property>
 
   <property>
     <name>hive.auto.convert.join.noconditionaltask.size</name>
     <value>1000000000</value>
+    <description>If hive.auto.convert.join.noconditionaltask is off, this parameter
does not take affect. However, if it
+      is on, and the sum of size for n-1 of the tables/partitions for a n-way join is smaller
than this size, the join is directly
+      converted to a mapjoin(there is no conditional task). The default is 10MB.
+    </description>
   </property>
 
   <property>
     <name>hive.optimize.reducededuplication.min.reducer</name>
     <value>1</value>
+    <description>Reduce deduplication merges two RSs by moving key/parts/reducer-num
of the child RS to parent RS.
+      That means if reducer-num of the child RS is fixed (order by or forced bucketing) and
small, it can make very slow, single MR.
+      The optimization will be disabled if number of reducers is less than specified value.
+    </description>
   </property>
 
   <property>
     <name>hive.optimize.mapjoin.mapreduce</name>
     <value>true</value>
+    <description>If hive.auto.convert.join is off, this parameter does not take
+      affect. If it is on, and if there are map-join jobs followed by a map-reduce
+      job (for e.g a group by), each map-only job is merged with the following
+      map-reduce job.
+    </description>
   </property>
 
 </configuration>



Mime
View raw message