ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vbrodets...@apache.org
Subject ambari git commit: AMBARI-20069. hive.enforce.bucketing=true.(vbrodetskyi)
Date Tue, 21 Feb 2017 01:12:17 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 1ecdee0d5 -> 60aaaeab5


AMBARI-20069. hive.enforce.bucketing=true.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60aaaeab
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60aaaeab
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60aaaeab

Branch: refs/heads/trunk
Commit: 60aaaeab5e2e9c7581f982525950bf6290db3018
Parents: 1ecdee0
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Tue Feb 21 02:15:11 2017 +0200
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Tue Feb 21 02:15:11 2017 +0200

----------------------------------------------------------------------
 .../main/resources/stacks/HDP/2.2/services/stack_advisor.py   | 7 +++++--
 .../main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml | 5 +++++
 .../stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml        | 6 ++++++
 .../main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml    | 2 ++
 .../main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml | 5 +++++
 .../stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml        | 6 ++++++
 .../main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml    | 2 ++
 .../main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 +++++
 .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml        | 6 ++++++
 .../main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml    | 1 +
 .../src/test/python/stacks/2.2/common/test_stack_advisor.py   | 1 -
 .../src/test/python/stacks/2.3/common/test_stack_advisor.py   | 2 --
 12 files changed, 43 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
index 0d7b8b9..ede41fc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py
@@ -374,14 +374,12 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       putHiveSiteProperty("hive.support.concurrency", "true")
       putHiveSiteProperty("hive.compactor.initiator.on", "true")
       putHiveSiteProperty("hive.compactor.worker.threads", "1")
-      putHiveSiteProperty("hive.enforce.bucketing", "true")
       putHiveSiteProperty("hive.exec.dynamic.partition.mode", "nonstrict")
     else:
       putHiveSiteProperty("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager")
       putHiveSiteProperty("hive.support.concurrency", "false")
       putHiveSiteProperty("hive.compactor.initiator.on", "false")
       putHiveSiteProperty("hive.compactor.worker.threads", "0")
-      putHiveSiteProperty("hive.enforce.bucketing", "false")
       putHiveSiteProperty("hive.exec.dynamic.partition.mode", "strict")
 
     hiveMetastoreHost = self.getHostWithComponent("HIVE", "HIVE_METASTORE", services, hosts)
@@ -1497,6 +1495,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
             ldap_domain_property + " property, if you are using AD, if not, then " + ldap_baseDN_property
+ "!")})
 
 
+    hive_enforce_bucketing = "hive.enforce.bucketing"
+    if hive_enforce_bucketing in properties and properties[hive_enforce_bucketing].lower()
== "false":
+      validationItems.append({"config-name" : hive_enforce_bucketing, "item" :
+        self.getWarnItem("Set " + hive_enforce_bucketing + " to true otherwise there is a
potential of data corruption!")})
+
     configurationValidationProblems = self.toConfigurationValidationProblems(validationItems,
"hive-site")
     configurationValidationProblems.extend(parentValidationProblems)
     return configurationValidationProblems

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index ba155ab..cdd701a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -195,6 +195,11 @@
             <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)"
replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
             <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender"
replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.DRFA.MaxFileSize
= {{hive_log_maxfilesize}}MB"/>
           </definition>
+
+          <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+            <type>hive-site</type>
+            <set key="hive.enforce.bucketing" value="true"/>
+          </definition>
         </changes>
       </component>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index e6a3c94..ff42022 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -419,6 +419,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing
property">
+        <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+          <summary>Updating hive.enforce.bucketing property to true value</summary>
+        </task>
+      </execute-stage>
+
       <!--OOZIE-->
       <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Apply config changes
for Oozie Server">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.OozieConfigCalculation">

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 7d8438f..b255bc0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -893,6 +893,8 @@
             <summary>Update hive-env content</summary>
           </task>
           <task xsi:type="configure" id="hive_log4j_parameterize" />
+
+          <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"
/>
         </pre-upgrade>
 
         <pre-downgrade/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index d19bb76..1bea263 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -96,6 +96,11 @@
             <regex-replace key="content" find="#log4j.appender.DRFA.MaxBackupIndex=([0-9]+)"
replace-with="#log4j.appender.DRFA.MaxBackupIndex={{hive_log_maxbackupindex}}"/>
             <replace key="content" find="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender"
replace-with="log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender&#xA;log4j.appender.DRFA.MaxFileSize
= {{hive_log_maxfilesize}}MB"/>
           </definition>
+
+          <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+            <type>hive-site</type>
+            <set key="hive.enforce.bucketing" value="true"/>
+          </definition>
          </changes>
       </component>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index 69a894c..86cd56a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -461,6 +461,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing
property">
+        <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+          <summary>Updating hive.enforce.bucketing property to true value</summary>
+        </task>
+      </execute-stage>
+
       <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Parameterizing WebHCat
Log4J Properties">
         <task xsi:type="configure" id="webhcat_log4j_parameterize">
           <summary>Updating the Webhcat Log4J properties to include parameterizations</summary>

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index 0c9a8ea..6a8e9d7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -892,6 +892,8 @@
           </task>
           <task xsi:type="configure" id="hive_log4j_parameterize" />
 
+          <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"
/>
+
         </pre-upgrade>
 
         <pre-downgrade/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 241ca8f..ca9cf47 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -358,6 +358,11 @@
             <regex-replace key="content" find="property.llap.daemon.log.maxfilesize =
([0-9]+)MB" replace-with="property.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB"/>
             <regex-replace key="content" find="property.llap.daemon.log.maxbackupindex
= ([0-9]+)" replace-with="property.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}"/>
           </definition>
+
+          <definition xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+            <type>hive-site</type>
+            <set key="hive.enforce.bucketing" value="true"/>
+          </definition>
         </changes>
       </component>
       <component name="HIVE_SERVER_INTERACTIVE">

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 0ef69cb..06cae78 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -478,6 +478,12 @@
         <task xsi:type="configure" id="hdp_2_6_0_0_tez_append_heap_dump_options_for_tez_am"/>
       </execute-stage>
 
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Set hive.enforce.bucketing
property">
+        <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property">
+          <summary>Updating hive.enforce.bucketing property to true value</summary>
+        </task>
+      </execute-stage>
+
       <execute-stage service="HIVE" component="HIVE_SERVER_INTERACTIVE" title="Appending
heap dump options for HiveSever2 Interactive">
         <task xsi:type="configure" id="hdp_2_6_0_0_hive_llap_append_heap_dump_options"/>
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 6309bcb..64edbb8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -781,6 +781,7 @@
           <task xsi:type="configure" id="hdp_2_6_0_0_hive_append_heap_dump_options"/>
           <task xsi:type="configure" id="hive_log4j_parameterize" />
           <task xsi:type="configure" id="hive_llap_log4j_parameterize" />
+          <task xsi:type="configure" id="hdp_2_6_0_0_hive_set_hive_enforce_bucketing_property"
/>
         </pre-upgrade>
         
         <pre-downgrade />

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
index 0c7af1a..b8762b3 100644
--- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py
@@ -1300,7 +1300,6 @@ class TestHDP22StackAdvisor(TestCase):
           'hive.compactor.initiator.on': 'false',
           'hive.compactor.worker.threads': '0',
           'hive.compute.query.using.stats': 'true',
-          'hive.enforce.bucketing': 'false',
           'hive.exec.dynamic.partition.mode': 'strict',
           'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
           'hive.exec.orc.compression.strategy': 'SPEED',

http://git-wip-us.apache.org/repos/asf/ambari/blob/60aaaeab/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 7e93815..443331a 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -605,7 +605,6 @@ class TestHDP23StackAdvisor(TestCase):
           'hive.compactor.initiator.on': 'false',
           'hive.compactor.worker.threads': '0',
           'hive.compute.query.using.stats': 'true',
-          'hive.enforce.bucketing': 'false',
           'hive.exec.dynamic.partition.mode': 'strict',
           'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
           'hive.exec.orc.compression.strategy': 'SPEED',
@@ -883,7 +882,6 @@ class TestHDP23StackAdvisor(TestCase):
           'hive.compactor.initiator.on': 'false',
           'hive.compactor.worker.threads': '0',
           'hive.compute.query.using.stats': 'true',
-          'hive.enforce.bucketing': 'false',
           'hive.exec.dynamic.partition.mode': 'strict',
           'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
           'hive.exec.orc.compression.strategy': 'SPEED',


Mime
View raw message