ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-15785. HiveServerInteractive. Kerberos Support.
Date Mon, 11 Apr 2016 20:33:07 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 6ea7fd750 -> 9f61db3d1


AMBARI-15785. HiveServerInteractive. Kerberos Support.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9f61db3d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9f61db3d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9f61db3d

Branch: refs/heads/trunk
Commit: 9f61db3d12921dabf7572e708206b186dc8f5ffe
Parents: 6ea7fd7
Author: sshridhar <sshridhar@hortonworks.com>
Authored: Sun Apr 10 20:08:42 2016 -0700
Committer: swapan <swapan@apache.org>
Committed: Mon Apr 11 13:33:02 2016 -0700

----------------------------------------------------------------------
 .../package/scripts/hive_server_interactive.py  |  48 +++-
 .../0.12.0.2.0/package/scripts/params_linux.py  |   4 +-
 .../configuration/hive-interactive-site.xml     |  16 ++
 .../stacks/HDP/2.5/services/HIVE/kerberos.json  | 125 ++++++++++
 .../stacks/HDP/2.5/services/YARN/kerberos.json  | 243 +++++++++++++++++++
 .../stacks/HDP/2.5/services/stack_advisor.py    |   9 +
 6 files changed, 441 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 055296b..d909f6d 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -34,11 +34,13 @@ from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from ambari_commons import OSCheck, OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
+from pwd import getpwnam
 
 # Local Imports
 from setup_ranger_hive import setup_ranger_hive
 from hive_service_interactive import hive_service_interactive
 from hive_interactive import hive_interactive
+from hive_server import HiveServerDefault
 
 
 class HiveServerInteractive(Script):
@@ -74,6 +76,10 @@ class HiveServerDefault(HiveServerInteractive):
       env.set_params(params)
       self.configure(env)
 
+      if params.security_enabled:
+        # Do the security setup, internally calls do_kinit()
+        self.setup_security()
+
       # TODO : We need have conditional [re]start of LLAP once "status check command" for
LLAP is ready.
       # Check status and based on that decide on [re]starting.
 
@@ -87,11 +93,15 @@ class HiveServerDefault(HiveServerInteractive):
         hive_service_interactive('hiveserver2', action='start', upgrade_type=upgrade_type)
       else:
         Logger.info("Skipping start of Hive Server Interactive due to LLAP start issue.")
+        raise Exception("Problem starting HiveServer2")
 
     def stop(self, env, upgrade_type=None):
       import params
       env.set_params(params)
 
+      if params.security_enabled:
+        self.do_kinit()
+
       # TODO, why must Hive Server Interactive be stopped before LLAP???
 
       # Stop Hive Interactive Server first
@@ -114,7 +124,7 @@ class HiveServerDefault(HiveServerInteractive):
       pass
 
     def security_status(self, env):
-      pass
+      HiveServerDefault.security_status(env)
 
     def restart_llap(self, env):
       """
@@ -123,6 +133,10 @@ class HiveServerDefault(HiveServerInteractive):
       Logger.info("Custom Command to retart LLAP")
       import params
       env.set_params(params)
+
+      if params.security_enabled:
+        self.do_kinit();
+
       self._llap_stop(env)
       self._llap_start(env)
 
@@ -153,6 +167,9 @@ class HiveServerDefault(HiveServerInteractive):
           message += " " + error
         raise Fail(message)
 
+    """
+    Controls the start of LLAP.
+    """
     def _llap_start(self, env, cleanup=False):
       import params
       env.set_params(params)
@@ -162,7 +179,14 @@ class HiveServerDefault(HiveServerInteractive):
       # TODO : Currently hardcoded the params. Need to read the suggested values from hive2/hive-site.xml.
       # TODO, ensure that script works as hive from cmd when not cd'ed in /homve/hive
       # Needs permission to write to hive home dir.
-      cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances
1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO")
+
+      cmd = ''
+      if params.security_enabled:
+        cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances
1 -slider-am-container-mb "
+                     "{slider_am_container_mb} --slider-keytab-dir .slider/keytabs/{params.hive_user}/
--slider-keytab "
+                     "{hive_llap_keytab_file} --slider-principal {hive_headless_keytab} --loglevel
INFO")
+      else:
+        cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances
1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO")
 
       run_file_path = None
       try:
@@ -184,7 +208,6 @@ class HiveServerDefault(HiveServerInteractive):
           raise Fail("Did not find run.sh file in output: " + str(output))
 
         Logger.info(format("Run file path: {run_file_path}"))
-
         if os.path.isfile(run_file_path):
           Execute(run_file_path, user=params.hive_user)
 
@@ -193,6 +216,7 @@ class HiveServerDefault(HiveServerInteractive):
           Logger.info("Sleeping for 30 secs")
           time.sleep(30)
           Logger.info("LLAP app deployed successfully.")
+          return True
         else:
           raise Fail(format("Did not find run file {run_file_path}"))
       except:
@@ -207,7 +231,25 @@ class HiveServerDefault(HiveServerInteractive):
 
         # throw the original exception
         raise
+      return False
+
+    """
+    Does kinit and copies keytab for Hive/LLAP to HDFS.
+    """
+    def setup_security(self):
+      import params
+
+      self.do_kinit();
+
+      # Copy params.hive_llap_keytab_file to hdfs://<host>:<port>/user/<hive_user>/.slider/keytabs/<hive_user>
, required by LLAP
+      slider_keytab_install_cmd = format("slider install-keytab --keytab {params.hive_llap_keytab_file}
--folder {params.hive_user} --overwrite")
+      Execute(slider_keytab_install_cmd, user=params.hive_user)
+
+    def do_kinit(self):
+      import params
 
+      hive_interactive_kinit_cmd = format("{kinit_path_local} -kt {hive_server2_keytab} {hive_principal};
")
+      Execute(hive_interactive_kinit_cmd, user=params.hive_user)
 
 if __name__ == "__main__":
     HiveServerInteractive().execute()
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index c7d57c0..f878141 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -498,7 +498,9 @@ if has_hive_interactive:
   # Tez for Hive interactive related
   tez_interactive_config_dir = os.path.realpath("/etc/tez_hive2/conf")
   tez_interactive_user = config['configurations']['tez-env']['tez_user']
-
+  if security_enabled:
+    hive_llap_keytab_file = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.keytab.file']
+    hive_headless_keytab = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.principal']
 
 # ranger host
 ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])

http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
index 5830a52..ba46f00 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml
@@ -472,4 +472,20 @@ limitations under the License.
     <value>true</value>
   </property>
 
+  <property>
+    <name>hive.llap.daemon.work.dirs</name>
+    <value>${yarn.nodemanager.local-dirs}</value>
+  </property>
+
+  <property>
+    <name>hive.llap.zk.sm.connectionString</name>
+    <value>localhost:2181</value>
+    <depends-on>
+      <property>
+        <type>zoo.cfg</type>
+        <name>clientPort</name>
+      </property>
+    </depends-on>
+  </property>
+
 </configuration>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
new file mode 100644
index 0000000..fe0c2ec
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
@@ -0,0 +1,125 @@
+{
+  "services": [
+    {
+      "name": "HIVE",
+      "identities": [
+        {
+          "name": "/spnego"
+        },
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "configurations": [
+        {
+          "hive-site": {
+            "hive.metastore.sasl.enabled": "true",
+            "hive.server2.authentication": "KERBEROS"
+          }
+        },
+        {
+          "webhcat-site": {
+            "templeton.kerberos.secret": "secret",
+            "templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083,
\\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
+          }
+        },
+        {
+          "core-site": {
+            "hadoop.proxyuser.HTTP.hosts": "${clusterHostInfo/webhcat_server_host}"
+          }
+        },
+        {
+          "hive-interactive-site": {
+            "hadoop.security.authentication": "KERBEROS",
+            "hadoop.security.authorization": "true"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "HIVE_METASTORE",
+          "identities": [
+            {
+              "name": "/HIVE/HIVE_SEVER/hive_server_hive",
+              "principal": {
+                "configuration": "hive-site/hive.metastore.kerberos.principal"
+              },
+              "keytab": {
+                "configuration": "hive-site/hive.metastore.kerberos.keytab.file"
+              }
+            }
+          ]
+        },
+        {
+          "name": "HIVE_SERVER",
+          "identities": [
+            {
+              "name": "/HDFS/NAMENODE/hdfs"
+            },
+            {
+              "name": "hive_server_hive",
+              "principal": {
+                "value": "hive/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "hive-site/hive.server2.authentication.kerberos.principal",
+                "local_username": "${hive-env/hive_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/hive.service.keytab",
+                "owner": {
+                  "name": "${hive-env/hive_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": "r"
+                },
+                "configuration": "hive-site/hive.server2.authentication.kerberos.keytab"
+              }
+            },
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "hive-site/hive.server2.authentication.spnego.principal"
+              },
+              "keytab": {
+                "configuration": "hive-site/hive.server2.authentication.spnego.keytab"
+              }
+            }
+          ]
+        },
+        {
+          "name": "HIVE_SERVER_INTERACTIVE",
+          "identities": [
+            {
+              "name": "/HDFS/NAMENODE/hdfs"
+            },
+            {
+              "name": "/HIVE/HIVE_SERVER/hive_server_hive"
+            },
+            {
+              "name": "/HIVE/HIVE_SERVER/spnego"
+            },
+            {
+              "name": "/YARN/NODEMANAGER/llap_zk_hive"
+            }
+          ]
+        },
+        {
+          "name": "WEBHCAT_SERVER",
+          "identities": [
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "webhcat-site/templeton.kerberos.principal"
+              },
+              "keytab": {
+                "configuration": "webhcat-site/templeton.kerberos.keytab"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
new file mode 100644
index 0000000..367dfd8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/kerberos.json
@@ -0,0 +1,243 @@
+{
+  "services": [
+    {
+      "name": "YARN",
+      "identities": [
+        {
+          "name": "/spnego"
+        },
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "configurations": [
+        {
+          "yarn-site": {
+            "yarn.timeline-service.enabled": "false",
+            "yarn.timeline-service.http-authentication.type": "kerberos",
+            "yarn.acl.enable": "true",
+            "yarn.timeline-service.http-authentication.signature.secret": "",
+            "yarn.timeline-service.http-authentication.signature.secret.file": "",
+            "yarn.timeline-service.http-authentication.signer.secret.provider": "",
+            "yarn.timeline-service.http-authentication.signer.secret.provider.object": "",
+            "yarn.timeline-service.http-authentication.token.validity": "",
+            "yarn.timeline-service.http-authentication.cookie.domain": "",
+            "yarn.timeline-service.http-authentication.cookie.path": "",
+            "yarn.timeline-service.http-authentication.proxyusers.*.hosts": "",
+            "yarn.timeline-service.http-authentication.proxyusers.*.users": "",
+            "yarn.timeline-service.http-authentication.proxyusers.*.groups": "",
+            "yarn.timeline-service.http-authentication.kerberos.name.rules": "",
+            "yarn.resourcemanager.proxyusers.*.groups": "",
+            "yarn.resourcemanager.proxyusers.*.hosts": "",
+            "yarn.resourcemanager.proxyusers.*.users": "",
+            "yarn.resourcemanager.proxy-user-privileges.enabled": "true",
+            "yarn.nodemanager.linux-container-executor.cgroups.mount-path": ""
+          }
+        },
+        {
+          "core-site": {
+            "hadoop.proxyuser.${yarn-env/yarn_user}.groups": "*",
+            "hadoop.proxyuser.${yarn-env/yarn_user}.hosts": "${clusterHostInfo/rm_host}"
+          }
+        }
+      ],
+      "components": [
+        {
+          "name": "NODEMANAGER",
+          "identities": [
+            {
+              "name": "nodemanager_nm",
+              "principal": {
+                "value": "nm/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "yarn-site/yarn.nodemanager.principal",
+                "local_username": "${yarn-env/yarn_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/nm.service.keytab",
+                "owner": {
+                  "name": "${yarn-env/yarn_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "yarn-site/yarn.nodemanager.keytab"
+              }
+            },
+            {
+              "name": "/HIVE/HIVE_SERVER/hive_server_hive",
+              "principal": {
+                "configuration": "hive-interactive-site/hive.llap.daemon.service.principal"
+              },
+              "keytab": {
+                "configuration": "hive-interactive-site/hive.llap.daemon.keytab.file"
+              }
+            },
+            {
+              "name": "llap_zk_hive",
+              "principal": {
+                "value": "hive@${realm}",
+                "type" : "user",
+                "configuration": "hive-interactive-site/hive.llap.zk.sm.principal"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/hive.llap.zk.sm.keytab",
+                "owner": {
+                  "name": "${yarn-env/yarn_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": "r"
+                },
+                "configuration": "hive-interactive-site/hive.llap.zk.sm.keytab.file"
+              }
+            },
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-principal"
+              },
+              "keytab": {
+                "configuration": "yarn-site/yarn.nodemanager.webapp.spnego-keytab-file"
+              }
+            }
+          ],
+          "configurations": [
+            {
+              "yarn-site": {
+                "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor"
+              }
+            }
+          ]
+        },
+        {
+          "name": "RESOURCEMANAGER",
+          "identities": [
+            {
+              "name": "resource_manager_rm",
+              "principal": {
+                "value": "rm/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "yarn-site/yarn.resourcemanager.principal",
+                "local_username": "${yarn-env/yarn_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/rm.service.keytab",
+                "owner": {
+                  "name": "${yarn-env/yarn_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "yarn-site/yarn.resourcemanager.keytab"
+              }
+            },
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-principal"
+              },
+              "keytab": {
+                "configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file"
+              }
+            }
+          ]
+        },
+        {
+          "name": "APP_TIMELINE_SERVER",
+          "identities": [
+            {
+              "name": "app_timeline_server_yarn",
+              "principal": {
+                "value": "yarn/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "yarn-site/yarn.timeline-service.principal",
+                "local_username": "${yarn-env/yarn_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/yarn.service.keytab",
+                "owner": {
+                  "name": "${yarn-env/yarn_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "yarn-site/yarn.timeline-service.keytab"
+              }
+            },
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.principal"
+              },
+              "keytab": {
+                "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab"
+              }
+            },
+            {
+              "name": "/HDFS/NAMENODE/hdfs"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "name": "MAPREDUCE2",
+      "identities": [
+        {
+          "name": "/spnego"
+        },
+        {
+          "name": "/smokeuser"
+        }
+      ],
+      "components": [
+        {
+          "name": "HISTORYSERVER",
+          "identities": [
+            {
+              "name": "/HDFS/NAMENODE/hdfs"
+            },
+            {
+              "name": "history_server_jhs",
+              "principal": {
+                "value": "jhs/_HOST@${realm}",
+                "type" : "service",
+                "configuration": "mapred-site/mapreduce.jobhistory.principal",
+                "local_username": "${mapred-env/mapred_user}"
+              },
+              "keytab": {
+                "file": "${keytab_dir}/jhs.service.keytab",
+                "owner": {
+                  "name": "${mapred-env/mapred_user}",
+                  "access": "r"
+                },
+                "group": {
+                  "name": "${cluster-env/user_group}",
+                  "access": ""
+                },
+                "configuration": "mapred-site/mapreduce.jobhistory.keytab"
+              }
+            },
+            {
+              "name": "/spnego",
+              "principal": {
+                "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-principal"
+              },
+              "keytab": {
+                "configuration": "mapred-site/mapreduce.jobhistory.webapp.spnego-keytab-file"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/9f61db3d/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index 1812d05..7407da3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -121,6 +121,15 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       pass
     pass
 
+    if 'hive-interactive-site' in services['configurations'] and \
+      services['configurations']['hive-interactive-site']['properties']['hive.llap.zk.sm.connectionString']:
+
+      # Fill the property 'hive.llap.zk.sm.connectionString' required by Hive Server Interactive
(HiveServer2)
+      zookeeper_host_port = self.getZKHostPortString(services)
+      if zookeeper_host_port:
+        putHiveInteractiveSiteProperty = self.putProperty(configurations, "hive-interactive-site",
services)
+        putHiveInteractiveSiteProperty("hive.llap.zk.sm.connectionString", zookeeper_host_port)
+
 
   def recommendRangerConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP25StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData,
services, hosts)


Mime
View raw message