Repository: bigtop Updated Branches: refs/heads/master 3ee4257de -> 86f8929f0 BIGTOP-1872: Improve HUE puppet recipes Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/86f8929f Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/86f8929f Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/86f8929f Branch: refs/heads/master Commit: 86f8929f0f9b0f3cf3a19eb77f27b8f4db7ffb22 Parents: 3ee4257 Author: Olaf Flebbe Authored: Wed May 13 22:15:34 2015 +0200 Committer: Olaf Flebbe Committed: Tue May 19 21:23:28 2015 +0200 ---------------------------------------------------------------------- .../puppet/modules/hue/manifests/init.pp | 28 +- .../puppet/modules/hue/templates/hue.ini | 276 ++++++++----------- 2 files changed, 133 insertions(+), 171 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/86f8929f/bigtop-deploy/puppet/modules/hue/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp index 9974cb0..4fd9c1c 100644 --- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp @@ -14,13 +14,22 @@ # limitations under the License. class hue { - class server($sqoop_url, $solr_url, $hbase_thrift_url, - $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_url, $rm_proxy_url, $history_server_url, - $hue_host = "0.0.0.0", $hue_port = "8888", $default_fs = "hdfs://localhost:8020", - $kerberos_realm = "", $hue_apps = "all") { + class server($sqoop_url = "http://localhost:12000/sqoop", $solr_url = "http://localhost:8983/solr/", $hbase_thrift_url = "", + $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_proxy_url, $history_server_url, + $hive_host = "", $hive_port = "10000", + $rm_logical_name = undef, $rm_api_port = "8088", $app_blacklist = "impala, security", + $hue_host = "0.0.0.0", $hue_port = "8888", $hue_timezone = "America/Los_Angeles", + $default_fs = "hdfs://localhost:8020", + $kerberos_realm = "", $kerberos_principal = "", $huecert = undef, $huekey = undef, + $auth_backend = "desktop.auth.backend.AllowFirstUserDjangoBackend", + $ldap_url = undef, $ldap_cert = undef, $use_start_tls = "true", + $base_dn = undef , $bind_dn = undef, $bind_password = undef, + $user_name_attr = undef, $user_filter = undef, + $group_member_attr = undef, $group_filter = undef, + $hue_apps = "all" ) { $hue_packages = $hue_apps ? { - "all" => [ "hue" ], # The hue metapackage requires all apps + "all" => [ "hue", "hue-server" ], # The hue metapackage requires all apps "none" => [ "hue-server" ], default => concat(prefix($hue_apps, "hue-"), [ "hue-server" ]) } @@ -29,7 +38,7 @@ class hue { require kerberos::client kerberos::host_keytab { "hue": spnego => false, - require => Package[$hue_packages], + require => Package["hue-server"], } } @@ -44,12 +53,11 @@ class hue { service { "hue": ensure => running, - require => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"] ], - subscribe => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"] ], + require => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"]], + subscribe => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"]], hasrestart => true, hasstatus => true, - } + } Kerberos::Host_keytab <| title == "hue" |> -> Service["hue"] - } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/86f8929f/bigtop-deploy/puppet/modules/hue/templates/hue.ini ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hue/templates/hue.ini b/bigtop-deploy/puppet/modules/hue/templates/hue.ini index 283c332..fe1b472 100644 --- a/bigtop-deploy/puppet/modules/hue/templates/hue.ini +++ b/bigtop-deploy/puppet/modules/hue/templates/hue.ini @@ -1,18 +1,3 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Hue configuration file # =================================== # @@ -31,21 +16,21 @@ [desktop] - send_dbug_messages=1 - - # To show database transactions, set database_logging to 1 - database_logging=0 +# send_dbug_messages=1# +# +# # To show database transactions, set database_logging to 1 +# database_logging=0 # Set this to a random string, the longer the better. # This is used for secure hashing in the session store. - secret_key=deaddeafdeaddeafdeaddeafdeaddeafdeaddeafdeaddeafdeaddeaf + secret_key=Geheim! # Webserver listens on this address and port http_host=<%= @hue_host %> http_port=<%= @hue_port %> # Time zone name - time_zone=America/Los_Angeles + time_zone=<%= @hue_timezone %> # Enable or disable Django debug mode. django_debug_mode=false @@ -63,8 +48,8 @@ ## django_email_backend=django.core.mail.backends.smtp.EmailBackend # Webserver runs as this user - server_user=hue - server_group=hue + ## server_user=hue + ## server_group=hue # This should be the Hue admin and proxy user ## default_user=hue @@ -80,10 +65,14 @@ ## cherrypy_server_threads=10 # Filename of SSL Certificate - ## ssl_certificate= + <%if @huecert %> + ssl_certificate=<%= @huecert %> + <% end -%> # Filename of SSL RSA Private Key - ## ssl_private_key= + <%if @huekey %> + ssl_private_key=<%= @huekey %> + <% end -%> # List of allowed and disallowed ciphers in cipher list format. # See http://www.openssl.org/docs/apps/ciphers.html for more information on cipher list format. @@ -99,7 +88,7 @@ # Help improve Hue with anonymous usage analytics. # Use Google Analytics to see how many times an application or specific section of an application is used, nothing more. - ## collect_usage=true + collect_usage=false # Support for HTTPS termination at the load-balancer level with SECURE_PROXY_SSL_HEADER. ## secure_proxy_ssl_header=false @@ -115,7 +104,7 @@ # Comma separated list of apps to not load at server startup. # e.g.: pig,zookeeper - ## app_blacklist= + app_blacklist=<%= @app_blacklist %> # The directory where to store the auditing logs. Auditing is disable if the value is empty. # e.g. /var/log/hue/audit.log @@ -156,10 +145,10 @@ # - libopenid.backend.OpenIDBackend # - liboauth.backend.OAuthBackend # (Support Twitter, Facebook, Google+ and Linkedin - ## backend=desktop.auth.backend.AllowFirstUserDjangoBackend + backend=<%= @auth_backend %> # The service to use when querying PAM. - ## pam_service=login + #pam_service=login # When using the desktop.auth.backend.RemoteUserDjangoBackend, this sets # the normalized name of the header that contains the remote user. @@ -171,9 +160,6 @@ # Defaults to HTTP_REMOTE_USER ## remote_user_header=HTTP_REMOTE_USER - # Synchronize a users groups when they login - ## sync_groups_on_login=false - # Ignore the case of usernames when searching for existing users. # Only supported in remoteUserDjangoBackend. ## ignore_username_case=false @@ -193,118 +179,105 @@ # ------------------------------------------------------------------- [[ldap]] +<% if @ldap_url %> # The search base for finding users and groups - ## base_dn="DC=mycompany,DC=com" + base_dn="<%= @base_dn %>" # URL of the LDAP server - ## ldap_url=ldap://auth.mycompany.com + ldap_url=<%= @ldap_url %> # A PEM-format file containing certificates for the CA's that # Hue will trust for authentication over TLS. # The certificate for the CA that signed the # LDAP server certificate must be included among these certificates. # See more here http://www.openldap.org/doc/admin24/tls.html. - ## ldap_cert= - ## use_start_tls=true +<% if @ldap_cert -%> + ldap_cert=<%= @ldap_cert %> +<% end -%> + use_start_tls=<%= @use_start_tls %> # Distinguished name of the user to bind as -- not necessary if the LDAP server # supports anonymous searches - ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com" +<% if @bind_dn -%> + # Distinguished name of the user to bind as -- not necessary if the LDAP server + # supports anonymous searches + bind_dn="<%= @bind_dn %>" # Password of the bind user -- not necessary if the LDAP server supports # anonymous searches - ## bind_password= + bind_password=<%= @bind_password %> +<% end -%> +<% if @user_filter -%> + # Use search bind authentication. + search_bind_authentication=true +<% else -%> # Pattern for searching for usernames -- Use for the parameter # For use when using LdapBackend for Hue authentication - ## ldap_username_pattern="uid=,ou=People,dc=mycompany,dc=com" + # ldap_username_pattern="uid=,ou=People,dc=mycompany,dc=com" + + search_bind_authentication=false +<% end -%> + # Execute this script to produce the bind user password. This will be used + # when `bind_password` is not set. + ## bind_password_script= # Create users in Hue when they try to login with their LDAP credentials # For use when using LdapBackend for Hue authentication - ## create_users_on_login = true + create_users_on_login = true + + # Synchronize a users groups when they login + ## sync_groups_on_login=false # Ignore the case of usernames when searching for existing users in Hue. - ## ignore_username_case=false + ignore_username_case=true # Force usernames to lowercase when creating new users from LDAP. ## force_username_lowercase=false - # Use search bind authentication. - ## search_bind_authentication=true - # Choose which kind of subgrouping to use: nested or suboordinate (deprecated). ## subgroups=suboordinate # Define the number of levels to search for nested members. ## nested_members_search_depth=10 - [[[users]]] + # Whether or not to follow referrals + ## follow_referrals=false - # Base filter for searching for users - ## user_filter="objectclass=*" + # Enable python-ldap debugging. + ## debug=false - # The username attribute in the LDAP schema - ## user_name_attr=sAMAccountName + # Sets the debug level within the underlying LDAP C lib. + ## debug_level=255 + # Possible values for trace_level are 0 for no logging, 1 for only logging the method calls with arguments, + # 2 for logging the method calls with arguments and the complete results and 9 for also logging the traceback of method calls. + ## trace_level=0 + + [[[users]]] +<% if @user_filter -%> + # Base filter for searching for users + user_filter="<%= @user_filter %>" +<% end -%> +<% if @user_name_attr -%> + # The username attribute in the LDAP schema + user_name_attr=<%= @user_name_attr %> +<% end -%> [[[groups]]] # Base filter for searching for groups - ## group_filter="objectclass=*" +<% if @group_filter -%> + group_filter="objectclass=groupOfEntries" +<% end -%> # The group name attribute in the LDAP schema ## group_name_attr=cn # The attribute of the group object which identifies the members of the group - ## group_member_attr=members - - [[[ldap_servers]]] - - ## [[[[mycompany]]]] - - # The search base for finding users and groups - ## base_dn="DC=mycompany,DC=com" - - # URL of the LDAP server - ## ldap_url=ldap://auth.mycompany.com - - # A PEM-format file containing certificates for the CA's that - # Hue will trust for authentication over TLS. - # The certificate for the CA that signed the - # LDAP server certificate must be included among these certificates. - # See more here http://www.openldap.org/doc/admin24/tls.html. - ## ldap_cert= - ## use_start_tls=true - - # Distinguished name of the user to bind as -- not necessary if the LDAP server - # supports anonymous searches - ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com" - - # Password of the bind user -- not necessary if the LDAP server supports - # anonymous searches - ## bind_password= - - # Pattern for searching for usernames -- Use for the parameter - # For use when using LdapBackend for Hue authentication - ## ldap_username_pattern="uid=,ou=People,dc=mycompany,dc=com" - - ## Use search bind authentication. - ## search_bind_authentication=true - - ## [[[[[users]]]]] - - # Base filter for searching for users - ## user_filter="objectclass=Person" - - # The username attribute in the LDAP schema - ## user_name_attr=sAMAccountName - - ## [[[[[groups]]]]] - - # Base filter for searching for groups - ## group_filter="objectclass=groupOfNames" - - # The username attribute in the LDAP schema - ## group_name_attr=cn +<% if @group_member_attr -%> + group_member_attr=<%= @group_member_attr %> +<% end -%> +<% end -%> # Configuration options for specifying the Desktop Database. For more info, # see http://docs.djangoproject.com/en/1.4/ref/settings/#database-engine @@ -361,8 +334,7 @@ # Default email address to use for various automated notification from Hue ## default_from_email=hue@localhost - -<% if @kerberos_realm != "" %> +<% if @kerberos_realm != "" -%> # Configuration options for Kerberos integration for secured Hadoop clusters # ------------------------------------------------------------------------ [[kerberos]] @@ -370,12 +342,11 @@ # Path to Hue's Kerberos keytab file hue_keytab=/etc/hue.keytab # Kerberos principal name for Hue - hue_principal=hue/<%= @fqdn %> + hue_principal=hue/<%= @fqdn %>@<%= @kerberos_realm %> # Path to kinit - kinit_path=<%= (@operatingsystem == 'ubuntu') ? '/usr/bin' : '/usr/kerberos/bin' %>/kinit -<% end %> - + kinit_path=<%= (@operatingsystem == 'ubuntu' || @operatingsystem == 'Debian') ? '/usr/bin' : '/usr/kerberos/bin' %>/kinit +<% end -%> # Configuration options for using OAuthBackend (core) login # ------------------------------------------------------------------------ [[oauth]] @@ -616,25 +587,30 @@ # Configuration for YARN (MR2) # ------------------------------------------------------------------------ [[yarn_clusters]] - - [[[default]]] +<% resourcemanager_hosts = Array(@rm_host) -%> +<% resourcemanager_hosts.each do |host| -%> + [[[<%= host %>]]] # Enter the host on which you are running the ResourceManager - resourcemanager_host=<%= @rm_host %> + resourcemanager_host=<%= host %> # The port where the ResourceManager IPC listens on - resourcemanager_port=<%= @rm_port %> + # resourcemanager_port=<% rm_port %> # Whether to submit jobs to this cluster submit_to=True - +<% if @rm_logical_name -%> # Resource Manager logical name (required for HA) - ## logical_name= + logical_name=<%= @rm_logical_name %> +<% else -%> + # Resource Manager logical name (required for HA) + # logical_name= +<% end -%> # Change this if your YARN cluster is Kerberos-secured security_enabled=<%= if (@kerberos_realm != "") ; "true" else "false" end %> # URL of the ResourceManager API - resourcemanager_api_url=<%= @rm_url %> + resourcemanager_api_url=http://<%= host %>:<%= @rm_api_port %> # URL of the ProxyServer API proxy_api_url=<%= @rm_proxy_url %> @@ -642,43 +618,12 @@ # URL of the HistoryServer API history_server_api_url=<%= @history_server_url %> - # HA support by specifying multiple clusters - # e.g. - - # [[[ha]]] - # Resource Manager logical name (required for HA) - # logical_name=my-rm-name + # URL of the NodeManager API + node_manager_api_url=http://localhost:8042 +<% end -%> # Configuration for MapReduce (MR1) # ------------------------------------------------------------------------ - [[mapred_clusters]] - - [[[default]]] - # Enter the host on which you are running the Hadoop JobTracker - ## jobtracker_host=localhost - - # The port where the JobTracker IPC listens on - ## jobtracker_port=8021 - - # JobTracker logical name for HA - ## logical_name= - - # Thrift plug-in port for the JobTracker - ## thrift_port=9290 - - # Whether to submit jobs to this cluster - submit_to=False - - # Change this if your MapReduce cluster is Kerberos-secured - ## security_enabled=false - - # HA support by specifying multiple clusters - # e.g. - - # [[[ha]]] - # Enter the logical name of the JobTrackers - # logical_name=my-jt-name - ########################################################################### # Settings to configure the Filebrowser app @@ -694,10 +639,9 @@ [liboozie] # The URL where the Oozie service runs on. This is required in order for - # users to submit jobs. Empty value disables the config check. + # users to submit jobs. oozie_url=<%= @oozie_url %> - # Requires FQDN in oozie_url if enabled security_enabled=<%= if (@kerberos_realm != "") ; "true" else "false" end %> # Location on HDFS where the workflows/coordinator are deployed when submitted. @@ -716,10 +660,10 @@ ## sample_data_dir=...thirdparty/sample_data # Location on HDFS where the oozie examples and workflows are stored. - remote_data_dir=/user/hue/oozie/workspaces + ## remote_data_dir=/user/hue/oozie/workspaces # Maximum of Oozie workflows or coodinators to retrieve in one API call. - oozie_jobs_count=100 + ## oozie_jobs_count=100 # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit. ## enable_cron_scheduling=true @@ -731,19 +675,24 @@ [beeswax] +<% if @hive_host -%> # Host where HiveServer2 is running. # If Kerberos security is enabled, use fully-qualified domain name (FQDN). - ## hive_server_host=localhost + hive_server_host=<%= @hive_host %> # Port where HiveServer2 Thrift server runs on. - ## hive_server_port=10000 + hive_server_port=<%= @hive_port %> # Hive configuration directory, where hive-site.xml is located - ## hive_conf_dir=/etc/hive/conf + hive_conf_dir=/etc/hive/conf # Timeout in seconds for thrift calls to Hive service ## server_conn_timeout=120 + # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs. + # If false, Hue will use the FetchResults() thrift call instead. + use_get_log_api=false + # Set a LIMIT clause when browsing a partitioned table. # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. ## browse_partitioned_table_limit=250 @@ -775,7 +724,7 @@ # Choose whether Hue should validate certificates received from the server. ## validate=true - +<% end -%> ########################################################################### # Settings to configure Pig @@ -797,8 +746,9 @@ # For autocompletion, fill out the librdbms section. # Sqoop server URL +<% if @sqoop_url != "" -%> server_url=<%= @sqoop_url %> - +<% end -%> ########################################################################### # Settings to configure Proxy @@ -856,8 +806,11 @@ [hbase] # Comma-separated list of HBase Thrift servers for clusters in the format of '(name|host:port)'. # Use full hostname with security. +<% if @hbase_thrift_url != "" -%> hbase_clusters=(Bigtop|<%= @hbase_thrift_url %>) - +<% else -%> + ## hbase_clusters=(Cluster|localhost:9090) +<% end -%> # HBase configuration directory, where hbase-site.xml is located. ## hbase_conf_dir=/etc/hbase/conf @@ -877,13 +830,14 @@ [search] # URL of the Solr Server +<% if @solr_url != "" -%> solr_url=<%= @solr_url %> - +<% end -%> # Requires FQDN in solr_url if enabled ## security_enabled=false ## Query sent when no term is entered - empty_query=*:* + ## empty_query=*:* ########################################################################### @@ -939,10 +893,10 @@ [[[default]]] # Zookeeper ensemble. Comma separated list of Host/Port. # e.g. localhost:2181,localhost:2182,localhost:2183 - ## host_ports=localhost:2181 + host_ports=localhost:2181 # The URL of the REST contrib service (required for znode browsing) - ## rest_url=http://localhost:9998 + rest_url=http://localhost:9998 ###########################################################################