ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lpus...@apache.org
Subject [09/50] [abbrv] ambari git commit: AMBARI-22188. Make hive server create directories related to replication (echekanskiy)
Date Thu, 12 Oct 2017 17:27:10 GMT
AMBARI-22188. Make hive server create directories related to replication (echekanskiy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a47e792
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a47e792
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a47e792

Branch: refs/heads/feature-branch-AMBARI-21307
Commit: 4a47e792ae6e4fe4fb67ca80c96c5e2054f7a9cd
Parents: 7341567
Author: Eugene Chekanskiy <echekanskiy@hortonworks.com>
Authored: Wed Oct 11 19:38:41 2017 +0300
Committer: Eugene Chekanskiy <echekanskiy@hortonworks.com>
Committed: Wed Oct 11 19:38:41 2017 +0300

----------------------------------------------------------------------
 .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 47 +++++++++++++++++++
 .../HIVE/0.12.0.2.0/package/scripts/hive.py     | 34 +++++++++++++-
 .../0.12.0.2.0/package/scripts/params_linux.py  |  4 ++
 .../HIVE/2.1.0.3.0/configuration/hive-site.xml  | 48 ++++++++++++++++++++
 .../HIVE/2.1.0.3.0/package/scripts/hive.py      | 32 +++++++++++++
 .../2.1.0.3.0/package/scripts/params_linux.py   |  4 ++
 6 files changed, 168 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
index 69d1c69..762530b 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml
@@ -471,4 +471,51 @@ limitations under the License.
     </depends-on>
     <on-ambari-upgrade add="false"/>
   </property>
+
+  <!-- Hive replication properties -->
+  <property>
+    <name>hive.metastore.dml.events</name>
+    <value/>
+    <description>If true, the metastore will be asked to fire events for DML operations</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.repl.cm.enabled</name>
+    <value/>
+    <description>Turn on ChangeManager, so delete files will go to cmrootdir.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.metastore.transactional.event.listeners</name>
+    <value/>
+    <description>A comma separated list of Java classes that implement the org.apache.hadoop.hive.metastore.MetaStoreEventListener
interface. Both the metastore event and corresponding listener method will be invoked in the
same JDO transaction.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.repl.cmrootdir</name>
+    <value/>
+    <description>Root dir for ChangeManager, used for deleted files.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.repl.rootdir</name>
+    <value/>
+    <description>HDFS root dir for all replication dumps.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 8e176b6..c4b34a5 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -245,7 +245,22 @@ def setup_hiveserver2():
                          owner=params.hive_user,
                          group=params.hdfs_user,
                          mode=0777) # Hive expects this dir to be writeable by everyone as
it is used as a temp dir
-    
+
+  if params.hive_repl_cmrootdir is not None:
+    params.HdfsResource(params.hive_repl_cmrootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 01777)
+  if params.hive_repl_rootdir is not None:
+    params.HdfsResource(params.hive_repl_rootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 0700)
+
   params.HdfsResource(None, action="execute")
   
 def setup_non_client():
@@ -310,6 +325,23 @@ def setup_metastore():
                  create_parents = True,
                  mode=0777)
 
+  if params.hive_repl_cmrootdir is not None:
+    params.HdfsResource(params.hive_repl_cmrootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 01777)
+  if params.hive_repl_rootdir is not None:
+    params.HdfsResource(params.hive_repl_rootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 0700)
+  if params.hive_repl_cmrootdir is not None or params.hive_repl_rootdir is not None:
+    params.HdfsResource(None, action="execute")
+
 def create_metastore_schema():
   import params
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 1d68ef4..16e1a71 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -871,3 +871,7 @@ if 'druid-common' in config['configurations'] \
 
 
 manage_hive_fsroot = default('/configurations/cluster-env/manage_hive_fsroot', True)
+
+# replication directories
+hive_repl_cmrootdir = default('/configurations/hive-site/hive.repl.cmrootdir', None)
+hive_repl_rootdir = default('/configurations/hive-site/hive.repl.rootdir', None)

http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
index edbbc9d..3c153b6 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/configuration/hive-site.xml
@@ -2001,4 +2001,52 @@ limitations under the License.
     </description>
     <on-ambari-upgrade add="false"/>
   </property>
+
+  <!-- Hive replication properties -->
+  <property>
+    <name>hive.metastore.dml.events</name>
+    <value/>
+    <description>If true, the metastore will be asked to fire events for DML operations</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.repl.cm.enabled</name>
+    <value/>
+    <description>Turn on ChangeManager, so delete files will go to cmrootdir.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.metastore.transactional.event.listeners</name>
+    <value/>
+    <description>A comma separated list of Java classes that implement the org.apache.hadoop.hive.metastore.MetaStoreEventListener
interface. Both the metastore event and corresponding listener method will be invoked in the
same JDO transaction.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.repl.cmrootdir</name>
+    <value/>
+    <description>Root dir for ChangeManager, used for deleted files.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>hive.repl.rootdir</name>
+    <value/>
+    <description>HDFS root dir for all replication dumps.</description>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="false"/>
+  </property>
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
index e7e92ec..22ff9fd 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/hive.py
@@ -245,6 +245,21 @@ def setup_hiveserver2():
                          group=params.hdfs_user,
                          mode=0777) # Hive expects this dir to be writeable by everyone as
it is used as a temp dir
 
+  if params.hive_repl_cmrootdir is not None:
+    params.HdfsResource(params.hive_repl_cmrootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 01777)
+  if params.hive_repl_rootdir is not None:
+    params.HdfsResource(params.hive_repl_rootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 0700)
+
   params.HdfsResource(None, action="execute")
 
 def setup_non_client():
@@ -300,6 +315,23 @@ def setup_metastore():
        content=StaticFile('startMetastore.sh')
   )
 
+  if params.hive_repl_cmrootdir is not None:
+    params.HdfsResource(params.hive_repl_cmrootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 01777)
+  if params.hive_repl_rootdir is not None:
+    params.HdfsResource(params.hive_repl_rootdir,
+                        type = "directory",
+                        action = "create_on_execute",
+                        owner = params.hive_user,
+                        group=params.user_group,
+                        mode = 0700)
+  if params.hive_repl_cmrootdir is not None or params.hive_repl_rootdir is not None:
+    params.HdfsResource(None, action="execute")
+
 def create_metastore_schema():
   import params
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4a47e792/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
index f6b676b..fb2c84a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py
@@ -848,3 +848,7 @@ ranger_hive_metastore_lookup = default('/configurations/ranger-hive-plugin-prope
 
 if security_enabled:
   hive_metastore_principal_with_host = hive_metastore_principal.replace('_HOST', hostname.lower())
+
+# replication directories
+hive_repl_cmrootdir = default('/configurations/hive-site/hive.repl.cmrootdir', None)
+hive_repl_rootdir = default('/configurations/hive-site/hive.repl.rootdir', None)


Mime
View raw message