ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-22162. Move out the druid configurations from hive-site to hive-interactive-site. (Slim Bouguerra via Swapan Shridhar)."
Date Mon, 09 Oct 2017 09:39:11 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.6 2eb3d78d9 -> 013b388f4


AMBARI-22162. Move out the druid configurations from hive-site to hive-interactive-site. (Slim
Bouguerra via Swapan Shridhar)."


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/013b388f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/013b388f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/013b388f

Branch: refs/heads/branch-2.6
Commit: 013b388f41c2c22fa832778c1667cee037f7a073
Parents: 2eb3d78
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Mon Oct 9 02:38:05 2017 -0700
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Mon Oct 9 02:38:14 2017 -0700

----------------------------------------------------------------------
 .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 217 ------------------
 .../configuration/hive-interactive-site.xml     | 225 +++++++++++++++++++
 .../stacks/HDP/2.6/services/stack_advisor.py    |  12 +-
 .../stacks/HDP/2.6/upgrades/config-upgrade.xml  |  21 +-
 4 files changed, 241 insertions(+), 234 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/013b388f/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index cec74e4..2510fda 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -447,223 +447,6 @@ limitations under the License.
     <on-ambari-upgrade add="false"/>
   </property>
 
-  <!-- Druid related properties -->
-  <property>
-    <name>hive.druid.broker.address.default</name>
-    <value>localhost:8082</value>
-    <description>Host name of druid router if any or broker</description>
-    <on-ambari-upgrade add="false"/>
-    <depends-on>
-      <property>
-        <type>druid-router</type>
-        <name>druid.port</name>
-      </property>
-    </depends-on>
-  </property>
-
-  <property>
-    <name>hive.druid.metadata.uri</name>
-    <value>jdbc:mysql://localhost:3355/druid</value>
-    <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description>
-    <on-ambari-upgrade add="false"/>
-    <depends-on>
-      <property>
-        <type>druid-common</type>
-        <name>druid.metadata.storage.connector.connectURI</name>
-      </property>
-    </depends-on>
-  </property>
-
-  <property>
-    <name>hive.druid.coordinator.address.default</name>
-    <value>localhost:8082</value>
-    <description>Host name of druid router if any or broker</description>
-    <on-ambari-upgrade add="false"/>
-    <depends-on>
-      <property>
-        <type>druid-coordinator</type>
-        <name>druid.port</name>
-      </property>
-    </depends-on>
-  </property>
-
-  <property>
-    <name>hive.druid.metadata.password</name>
-    <value>{{druid_metadata_password}}</value>
-    <property-type>PASSWORD</property-type>
-    <display-name>Druid Metadata Password</display-name>
-    <description>Druid meta data storage password</description>
-    <value-attributes>
-      <type>password</type>
-      <empty-value-valid>true</empty-value-valid>
-    </value-attributes>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
-   <property>
-    <name>hive.druid.metadata.username</name>
-    <value>druid</value>
-    <description>Username used to connect to druid metadata storage</description>
-    <on-ambari-upgrade add="false"/>
-     <depends-on>
-      <property>
-        <type>druid-common</type>
-        <name>druid.metadata.storage.connector.user</name>
-      </property>
-    </depends-on>
-  </property>
-
-  <property>
-    <name>hive.druid.indexer.segments.granularity</name>
-    <display-name>Default Granularity for the Druid segments</display-name>
-    <value-attributes>
-      <type>value-list</type>
-      <entries>
-        <entry>
-          <value>YEAR</value>
-        </entry>
-        <entry>
-          <value>MONTH</value>
-        </entry>
-        <entry>
-          <value>WEEK</value>
-        </entry>
-        <entry>
-          <value>DAY</value>
-        </entry>
-        <entry>
-          <value>HOUR</value>
-        </entry>
-        <entry>
-          <value>MINUTE</value>
-        </entry>
-        <entry>
-          <value>SECOND</value>
-        </entry>
-      </entries>
-    </value-attributes>
-    <value>MINUTE</value>
-    <description>Default Granularity for the segments created by the Druid storage
handler, this can be overridden per table using table property druid.segment.granularity </description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-   <property>
-    <name>hive.druid.indexer.partition.size.max</name>
-    <value>5000000</value>
-    <description>Maximum number of records per segment partition</description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-   <property>
-    <name>hive.druid.indexer.memory.rownum.max</name>
-    <value>75000</value>
-    <description>Maximum number of records in memory while storing data in Druid</description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
-  <property>
-    <name>hive.druid.select.distribute</name>
-    <value>true</value>
-    <description>If it is set to true, we distribute the execution of Druid Select
queries</description>
-    <on-ambari-upgrade add="false"/>
-    <value-attributes>
-      <type>boolean</type>
-    </value-attributes>
-  </property>
-
-  <property>
-    <name>hive.druid.basePersistDirectory</name>
-    <value></value>
-    <description>
-      Local temporary directory used to persist intermediate indexing state,
-      if empty (recommended) will default to JVM system property java.io.tmpdir.
-    </description>
-    <value-attributes>
-      <empty-value-valid>true</empty-value-valid>
-    </value-attributes>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
-  <property>
-    <name>hive.druid.storage.storageDirectory</name>
-    <value>{{druid_storage_dir}}</value>
-    <description>
-      Druid deep storage location for segments.
-    </description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-  <property>
-    <name>hive.druid.metadata.db.type</name>
-    <display-name>Druid metadata storage type </display-name>
-    <value-attributes>
-      <overridable>false</overridable>
-      <type>value-list</type>
-      <entries>
-        <entry>
-          <value>mysql</value>
-          <label>MYSQL</label>
-        </entry>
-        <entry>
-          <value>postgresql</value>
-          <label>POSTGRESQL</label>
-        </entry>
-      </entries>
-    </value-attributes>
-    <value>mysql</value>
-    <depends-on>
-      <property>
-        <type>druid-common</type>
-        <name>druid.metadata.storage.type</name>
-      </property>
-    </depends-on>
-    <description>Druid metadata storage type</description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
-  <property>
-    <name>hive.druid.passiveWaitTimeMs</name>
-    <value>30000</value>
-    <description>
-      Wait time in ms default to 30 seconds.
-    </description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
-  <property>
-    <name>hive.druid.working.directory</name>
-    <value>/tmp/druid-indexing</value>
-    <description>
-      Default hdfs working directory used to store some intermediate metadata.
-    </description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
-  <property>
-    <name>hive.druid.maxTries</name>
-    <value>5</value>
-    <description>
-      Maximum number of http call retries before giving up.
-    </description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
-  <property>
-    <name>hive.druid.bitmap.type</name>
-    <display-name>Druid metadata storage type </display-name>
-    <value-attributes>
-      <type>value-list</type>
-      <entries>
-        <entry>
-          <value>roaring</value>
-        </entry>
-        <entry>
-          <value>concise</value>
-        </entry>
-      </entries>
-    </value-attributes>
-    <value>roaring</value>
-    <description>Druid Coding algorithm use to encode the bitmaps</description>
-    <on-ambari-upgrade add="false"/>
-  </property>
-
   <!-- This property is removed in HDP 2.5 and higher. -->
   <property>
     <name>atlas.rest.address</name>

http://git-wip-us.apache.org/repos/asf/ambari/blob/013b388f/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
index aae2efa..64cef3e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-interactive-site.xml
@@ -124,4 +124,229 @@ limitations under the License.
     <on-ambari-upgrade add="false"/>
   </property>
 
+  <!-- Druid related properties -->
+  <property>
+    <name>hive.druid.broker.address.default</name>
+    <value>localhost:8082</value>
+    <description>Host name of druid router if any or broker</description>
+    <on-ambari-upgrade add="false"/>
+    <depends-on>
+      <property>
+        <type>druid-router</type>
+        <name>druid.port</name>
+      </property>
+    </depends-on>
+  </property>
+
+  <property>
+    <name>hive.druid.metadata.uri</name>
+    <value>jdbc:mysql://localhost:3355/druid</value>
+    <description>URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)</description>
+    <on-ambari-upgrade add="false"/>
+    <depends-on>
+      <property>
+        <type>druid-common</type>
+        <name>druid.metadata.storage.connector.connectURI</name>
+      </property>
+    </depends-on>
+  </property>
+
+  <property>
+    <name>hive.druid.coordinator.address.default</name>
+    <value>localhost:8082</value>
+    <description>Host name of druid router if any or broker</description>
+    <on-ambari-upgrade add="false"/>
+    <depends-on>
+      <property>
+        <type>druid-coordinator</type>
+        <name>druid.port</name>
+      </property>
+    </depends-on>
+  </property>
+
+  <property>
+    <name>hive.druid.metadata.password</name>
+    <value>{{druid_metadata_password}}</value>
+    <property-type>PASSWORD</property-type>
+    <display-name>Druid Metadata Password</display-name>
+    <description>Druid meta data storage password</description>
+    <value-attributes>
+      <type>password</type>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+   <property>
+    <name>hive.druid.metadata.username</name>
+    <value>druid</value>
+    <description>Username used to connect to druid metadata storage</description>
+    <on-ambari-upgrade add="false"/>
+     <depends-on>
+      <property>
+        <type>druid-common</type>
+        <name>druid.metadata.storage.connector.user</name>
+      </property>
+    </depends-on>
+  </property>
+
+  <property>
+    <name>hive.druid.indexer.segments.granularity</name>
+    <display-name>Default Granularity for the Druid segments</display-name>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>YEAR</value>
+        </entry>
+        <entry>
+          <value>MONTH</value>
+        </entry>
+        <entry>
+          <value>WEEK</value>
+        </entry>
+        <entry>
+          <value>DAY</value>
+        </entry>
+        <entry>
+          <value>HOUR</value>
+        </entry>
+        <entry>
+          <value>MINUTE</value>
+        </entry>
+        <entry>
+          <value>SECOND</value>
+        </entry>
+      </entries>
+    </value-attributes>
+    <value>DAY</value>
+    <description>Default Granularity for the segments created by the Druid storage
handler, this can be overridden per table using table property druid.segment.granularity </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+   <property>
+    <name>hive.druid.indexer.partition.size.max</name>
+    <value>1000000</value>
+    <description>Maximum number of records per segment partition</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+   <property>
+    <name>hive.druid.indexer.memory.rownum.max</name>
+    <value>75000</value>
+    <description>Maximum number of records in memory while storing data in Druid</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>hive.druid.select.distribute</name>
+    <value>true</value>
+    <description>If it is set to true, we distribute the execution of Druid Select
queries</description>
+    <on-ambari-upgrade add="false"/>
+    <value-attributes>
+      <type>boolean</type>
+    </value-attributes>
+  </property>
+
+  <property>
+    <name>hive.druid.basePersistDirectory</name>
+    <value></value>
+    <description>
+      Local temporary directory used to persist intermediate indexing state,
+      if empty (recommended) will default to JVM system property java.io.tmpdir.
+    </description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>hive.druid.storage.storageDirectory</name>
+    <value>{{druid_storage_dir}}</value>
+    <description>
+      Druid deep storage location for segments.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.druid.metadata.db.type</name>
+    <display-name>Druid metadata storage type </display-name>
+    <value-attributes>
+      <overridable>false</overridable>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>mysql</value>
+          <label>MYSQL</label>
+        </entry>
+        <entry>
+          <value>postgresql</value>
+          <label>POSTGRESQL</label>
+        </entry>
+      </entries>
+    </value-attributes>
+    <value>mysql</value>
+    <depends-on>
+      <property>
+        <type>druid-common</type>
+        <name>druid.metadata.storage.type</name>
+      </property>
+    </depends-on>
+    <description>Druid metadata storage type</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>hive.druid.passiveWaitTimeMs</name>
+    <value>30000</value>
+    <description>
+      Wait time in ms default to 30 seconds.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>hive.druid.working.directory</name>
+    <value>/tmp/druid-indexing</value>
+    <description>
+      Default hdfs working directory used to store some intermediate metadata.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>hive.druid.maxTries</name>
+    <value>5</value>
+    <description>
+      Maximum number of http call retries before giving up.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>hive.druid.bitmap.type</name>
+    <display-name>Druid metadata storage type </display-name>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>roaring</value>
+        </entry>
+        <entry>
+          <value>concise</value>
+        </entry>
+      </entries>
+    </value-attributes>
+    <value>roaring</value>
+    <description>Druid Coding algorithm use to encode the bitmaps</description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
+  <property>
+    <name>hive.druid.http.read.timeout</name>
+    <value>PT10M</value>
+    <description>
+      Maximum number of http call retries before giving up.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/013b388f/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index be07988..bba1e22 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -656,7 +656,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
     # druid is not in list of services to be installed
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
     if 'DRUID' in servicesList:
-        putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
+        putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site",
services)
         if 'druid-coordinator' in services['configurations']:
             component_hosts = self.getHostsWithComponent("DRUID", 'DRUID_COORDINATOR', services,
hosts)
             if component_hosts is not None and len(component_hosts) > 0:
@@ -693,11 +693,11 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
             else:
                 druid_metadata_user = ""
 
-        putHiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port)
-        putHiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port)
-        putHiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri)
-        putHiveSiteProperty('hive.druid.metadata.username', druid_metadata_user)
-        putHiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type)
+        putHiveInteractiveSiteProperty('hive.druid.broker.address.default', druid_broker_host_port)
+        putHiveInteractiveSiteProperty('hive.druid.coordinator.address.default', druid_coordinator_host_port)
+        putHiveInteractiveSiteProperty('hive.druid.metadata.uri', druid_metadata_uri)
+        putHiveInteractiveSiteProperty('hive.druid.metadata.username', druid_metadata_user)
+        putHiveInteractiveSiteProperty('hive.druid.metadata.db.type', druid_metadata_type)
 
 
   def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):

http://git-wip-us.apache.org/repos/asf/ambari/blob/013b388f/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 5b3f97d..5dbcf2d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -75,11 +75,19 @@
             <set key ="atlas.jaas.ticketBased-KafkaClient.option.useTicketCache" value="true"
               if-type="cluster-env" if-key="security_enabled" if-value="true"/>
           </definition>
+        </changes>
+      </component>
+      <component name="HIVE_SERVER_INTERACTIVE">
+        <changes>
+          <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update
additional LLAP-Tez settings">
+            <type>tez-interactive-site</type>
+            <set key="tez.runtime.shuffle.ssl.enable" value="false"/>
+          </definition>
           <definition xsi:type="configure" id="hdp_2_6_maint_druid_config_for_hive_hook"
summary="Updating druid hive related properties">
             <type>hive-site</type>
             <set key="hive.druid.metadata.password" value="{{druid_metadata_password}}"
if-type="druid-common"/>
-            <set key="hive.druid.indexer.segments.granularity" value="MINUTE" if-type="druid-common"/>
-            <set key="hive.druid.indexer.partition.size.max" value="5000000" if-type="druid-common"/>
+            <set key="hive.druid.indexer.segments.granularity" value="DAY" if-type="druid-common"/>
+            <set key="hive.druid.indexer.partition.size.max" value="1000000" if-type="druid-common"/>
             <set key="hive.druid.indexer.memory.rownum.max" value="75000" if-type="druid-common"/>
             <set key="hive.druid.select.distribute" value="true" if-type="druid-common"/>
             <set key="hive.druid.basePersistDirectory" value="" if-type="druid-common"/>
@@ -87,15 +95,6 @@
             <set key="hive.druid.passiveWaitTimeMs" value="30000" if-type="druid-common"/>
             <set key="hive.druid.working.directory" value="/tmp/druid-indexing" if-type="druid-common"/>
             <set key="hive.druid.bitmap.type" value="roaring" if-type="druid-common"/>
-
-          </definition>
-        </changes>
-      </component>
-      <component name="HIVE_SERVER_INTERACTIVE">
-        <changes>
-          <definition xsi:type="configure" id="llap_update_tez_shuffle_ssl_enable" summary="Update
additional LLAP-Tez settings">
-            <type>tez-interactive-site</type>
-            <set key="tez.runtime.shuffle.ssl.enable" value="false"/>
           </definition>
         </changes>
       </component>


Mime
View raw message