bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ofle...@apache.org
Subject bigtop git commit: BIGTOP-1872: Improve HUE puppet recipes
Date Tue, 19 May 2015 19:45:53 GMT
Repository: bigtop
Updated Branches:
  refs/heads/master 3ee4257de -> 86f8929f0


BIGTOP-1872: Improve HUE puppet recipes


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/86f8929f
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/86f8929f
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/86f8929f

Branch: refs/heads/master
Commit: 86f8929f0f9b0f3cf3a19eb77f27b8f4db7ffb22
Parents: 3ee4257
Author: Olaf Flebbe <of@oflebbe.de>
Authored: Wed May 13 22:15:34 2015 +0200
Committer: Olaf Flebbe <of@oflebbe.de>
Committed: Tue May 19 21:23:28 2015 +0200

----------------------------------------------------------------------
 .../puppet/modules/hue/manifests/init.pp        |  28 +-
 .../puppet/modules/hue/templates/hue.ini        | 276 ++++++++-----------
 2 files changed, 133 insertions(+), 171 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/86f8929f/bigtop-deploy/puppet/modules/hue/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
index 9974cb0..4fd9c1c 100644
--- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
@@ -14,13 +14,22 @@
 # limitations under the License.
 
 class hue {
-  class server($sqoop_url, $solr_url, $hbase_thrift_url,
-                $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_url, $rm_proxy_url, $history_server_url,
-                $hue_host = "0.0.0.0", $hue_port = "8888", $default_fs = "hdfs://localhost:8020",
-                $kerberos_realm = "", $hue_apps = "all") {
+  class server($sqoop_url = "http://localhost:12000/sqoop", $solr_url = "http://localhost:8983/solr/",
$hbase_thrift_url = "",
+               $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_proxy_url, $history_server_url,
+               $hive_host = "", $hive_port = "10000",
+               $rm_logical_name = undef, $rm_api_port = "8088", $app_blacklist = "impala,
security",
+               $hue_host = "0.0.0.0", $hue_port = "8888", $hue_timezone = "America/Los_Angeles",
+               $default_fs = "hdfs://localhost:8020",
+               $kerberos_realm = "", $kerberos_principal = "", $huecert = undef, $huekey
= undef,
+               $auth_backend = "desktop.auth.backend.AllowFirstUserDjangoBackend",
+               $ldap_url = undef, $ldap_cert = undef, $use_start_tls = "true",
+               $base_dn = undef , $bind_dn = undef, $bind_password = undef,
+               $user_name_attr = undef, $user_filter = undef,
+               $group_member_attr = undef, $group_filter = undef,
+               $hue_apps = "all" ) {
 
     $hue_packages = $hue_apps ? {
-      "all"     => [ "hue" ], # The hue metapackage requires all apps
+      "all"     => [ "hue", "hue-server" ], # The hue metapackage requires all apps
       "none"    => [ "hue-server" ],
       default   => concat(prefix($hue_apps, "hue-"), [ "hue-server" ])
     }
@@ -29,7 +38,7 @@ class hue {
       require kerberos::client
       kerberos::host_keytab { "hue":
         spnego => false,
-        require => Package[$hue_packages],
+        require => Package["hue-server"],
       }
     }
 
@@ -44,12 +53,11 @@ class hue {
 
     service { "hue":
       ensure => running,
-      require => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"] ],
-      subscribe => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"] ],
+      require => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"]],
+      subscribe => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"]],
       hasrestart => true,
       hasstatus => true,
-    } 
+    }
     Kerberos::Host_keytab <| title == "hue" |> -> Service["hue"]
-
   }
 }

http://git-wip-us.apache.org/repos/asf/bigtop/blob/86f8929f/bigtop-deploy/puppet/modules/hue/templates/hue.ini
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hue/templates/hue.ini b/bigtop-deploy/puppet/modules/hue/templates/hue.ini
index 283c332..fe1b472 100644
--- a/bigtop-deploy/puppet/modules/hue/templates/hue.ini
+++ b/bigtop-deploy/puppet/modules/hue/templates/hue.ini
@@ -1,18 +1,3 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
 # Hue configuration file
 # ===================================
 #
@@ -31,21 +16,21 @@
 
 [desktop]
 
-  send_dbug_messages=1
-
-  # To show database transactions, set database_logging to 1
-  database_logging=0
+#  send_dbug_messages=1#
+#
+#  # To show database transactions, set database_logging to 1
+#  database_logging=0
 
   # Set this to a random string, the longer the better.
   # This is used for secure hashing in the session store.
-  secret_key=deaddeafdeaddeafdeaddeafdeaddeafdeaddeafdeaddeafdeaddeaf
+  secret_key=Geheim!
 
   # Webserver listens on this address and port
   http_host=<%= @hue_host %>
   http_port=<%= @hue_port %>
 
   # Time zone name
-  time_zone=America/Los_Angeles
+  time_zone=<%= @hue_timezone %>
 
   # Enable or disable Django debug mode.
   django_debug_mode=false
@@ -63,8 +48,8 @@
   ## django_email_backend=django.core.mail.backends.smtp.EmailBackend
 
   # Webserver runs as this user
-  server_user=hue
-  server_group=hue
+  ## server_user=hue
+  ## server_group=hue
 
   # This should be the Hue admin and proxy user
   ## default_user=hue
@@ -80,10 +65,14 @@
   ## cherrypy_server_threads=10
 
   # Filename of SSL Certificate
-  ## ssl_certificate=
+  <%if @huecert %>
+  ssl_certificate=<%= @huecert %>
+  <% end -%>
 
   # Filename of SSL RSA Private Key
-  ## ssl_private_key=
+  <%if @huekey %>
+  ssl_private_key=<%= @huekey %>
+  <% end -%>
 
   # List of allowed and disallowed ciphers in cipher list format.
   # See http://www.openssl.org/docs/apps/ciphers.html for more information on cipher list
format.
@@ -99,7 +88,7 @@
 
   # Help improve Hue with anonymous usage analytics.
   # Use Google Analytics to see how many times an application or specific section of an application
is used, nothing more.
-  ## collect_usage=true
+  collect_usage=false
 
   # Support for HTTPS termination at the load-balancer level with SECURE_PROXY_SSL_HEADER.
   ## secure_proxy_ssl_header=false
@@ -115,7 +104,7 @@
 
   # Comma separated list of apps to not load at server startup.
   # e.g.: pig,zookeeper
-  ## app_blacklist=
+  app_blacklist=<%= @app_blacklist %>
 
   # The directory where to store the auditing logs. Auditing is disable if the value is empty.
   # e.g. /var/log/hue/audit.log
@@ -156,10 +145,10 @@
     # - libopenid.backend.OpenIDBackend
     # - liboauth.backend.OAuthBackend
     #     (Support Twitter, Facebook, Google+ and Linkedin
-    ## backend=desktop.auth.backend.AllowFirstUserDjangoBackend
+    backend=<%= @auth_backend %>
 
     # The service to use when querying PAM.
-    ## pam_service=login
+    #pam_service=login
 
     # When using the desktop.auth.backend.RemoteUserDjangoBackend, this sets
     # the normalized name of the header that contains the remote user.
@@ -171,9 +160,6 @@
     # Defaults to HTTP_REMOTE_USER
     ## remote_user_header=HTTP_REMOTE_USER
 
-    # Synchronize a users groups when they login
-    ## sync_groups_on_login=false
-
     # Ignore the case of usernames when searching for existing users.
     # Only supported in remoteUserDjangoBackend.
     ## ignore_username_case=false
@@ -193,118 +179,105 @@
   # -------------------------------------------------------------------
   [[ldap]]
 
+<% if @ldap_url %>
     # The search base for finding users and groups
-    ## base_dn="DC=mycompany,DC=com"
+    base_dn="<%= @base_dn %>"
 
     # URL of the LDAP server
-    ## ldap_url=ldap://auth.mycompany.com
+    ldap_url=<%= @ldap_url %> 
 
     # A PEM-format file containing certificates for the CA's that
     # Hue will trust for authentication over TLS.
     # The certificate for the CA that signed the
     # LDAP server certificate must be included among these certificates.
     # See more here http://www.openldap.org/doc/admin24/tls.html.
-    ## ldap_cert=
-    ## use_start_tls=true
+<%     if @ldap_cert -%>
+    ldap_cert=<%= @ldap_cert %>
+<%     end -%>
+    use_start_tls=<%= @use_start_tls %>
 
     # Distinguished name of the user to bind as -- not necessary if the LDAP server
     # supports anonymous searches
-    ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com"
+<%     if @bind_dn -%>
+    # Distinguished name of the user to bind as -- not necessary if the LDAP server
+    # supports anonymous searches
+    bind_dn="<%= @bind_dn %>"
 
     # Password of the bind user -- not necessary if the LDAP server supports
     # anonymous searches
-    ## bind_password=
+    bind_password=<%= @bind_password %>
+<%     end -%>
 
+<%     if @user_filter -%>
+    # Use search bind authentication.
+    search_bind_authentication=true
+<%     else -%>
     # Pattern for searching for usernames -- Use <username> for the parameter
     # For use when using LdapBackend for Hue authentication
-    ## ldap_username_pattern="uid=<username>,ou=People,dc=mycompany,dc=com"
+    #    ldap_username_pattern="uid=<username>,ou=People,dc=mycompany,dc=com"
+
+    search_bind_authentication=false
+<%     end -%>
+    # Execute this script to produce the bind user password. This will be used
+    # when `bind_password` is not set.
+    ## bind_password_script=
 
     # Create users in Hue when they try to login with their LDAP credentials
     # For use when using LdapBackend for Hue authentication
-    ## create_users_on_login = true
+    create_users_on_login = true
+
+    # Synchronize a users groups when they login
+    ## sync_groups_on_login=false
 
     # Ignore the case of usernames when searching for existing users in Hue.
-    ## ignore_username_case=false
+    ignore_username_case=true
 
     # Force usernames to lowercase when creating new users from LDAP.
     ## force_username_lowercase=false
 
-    # Use search bind authentication.
-    ## search_bind_authentication=true
-
     # Choose which kind of subgrouping to use: nested or suboordinate (deprecated).
     ## subgroups=suboordinate
 
     # Define the number of levels to search for nested members.
     ## nested_members_search_depth=10
 
-    [[[users]]]
+    # Whether or not to follow referrals
+    ## follow_referrals=false
 
-      # Base filter for searching for users
-      ## user_filter="objectclass=*"
+    # Enable python-ldap debugging.
+    ## debug=false
 
-      # The username attribute in the LDAP schema
-      ## user_name_attr=sAMAccountName
+    # Sets the debug level within the underlying LDAP C lib.
+    ## debug_level=255
 
+    # Possible values for trace_level are 0 for no logging, 1 for only logging the method
calls with arguments,
+    # 2 for logging the method calls with arguments and the complete results and 9 for also
logging the traceback of method calls.
+    ## trace_level=0
+
+    [[[users]]]
+<%     if @user_filter -%>
+      # Base filter for searching for users
+      user_filter="<%= @user_filter %>"
+<%     end -%>
+<%     if @user_name_attr -%>
+      # The username attribute in the LDAP schema
+      user_name_attr=<%= @user_name_attr %>
+<%     end -%>
     [[[groups]]]
 
       # Base filter for searching for groups
-      ## group_filter="objectclass=*"
+<%     if @group_filter -%>
+      group_filter="objectclass=groupOfEntries"
+<%     end -%>
 
       # The group name attribute in the LDAP schema
       ## group_name_attr=cn
 
       # The attribute of the group object which identifies the members of the group
-      ## group_member_attr=members
-
-    [[[ldap_servers]]]
-
-      ## [[[[mycompany]]]]
-
-        # The search base for finding users and groups
-        ## base_dn="DC=mycompany,DC=com"
-
-        # URL of the LDAP server
-        ## ldap_url=ldap://auth.mycompany.com
-
-        # A PEM-format file containing certificates for the CA's that
-        # Hue will trust for authentication over TLS.
-        # The certificate for the CA that signed the
-        # LDAP server certificate must be included among these certificates.
-        # See more here http://www.openldap.org/doc/admin24/tls.html.
-        ## ldap_cert=
-        ## use_start_tls=true
-
-        # Distinguished name of the user to bind as -- not necessary if the LDAP server
-        # supports anonymous searches
-        ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com"
-
-        # Password of the bind user -- not necessary if the LDAP server supports
-        # anonymous searches
-        ## bind_password=
-
-        # Pattern for searching for usernames -- Use <username> for the parameter
-        # For use when using LdapBackend for Hue authentication
-        ## ldap_username_pattern="uid=<username>,ou=People,dc=mycompany,dc=com"
-
-        ## Use search bind authentication.
-        ## search_bind_authentication=true
-
-        ## [[[[[users]]]]]
-
-          # Base filter for searching for users
-          ## user_filter="objectclass=Person"
-
-          # The username attribute in the LDAP schema
-          ## user_name_attr=sAMAccountName
-
-        ## [[[[[groups]]]]]
-
-          # Base filter for searching for groups
-          ## group_filter="objectclass=groupOfNames"
-
-          # The username attribute in the LDAP schema
-          ## group_name_attr=cn
+<%     if @group_member_attr -%>
+      group_member_attr=<%= @group_member_attr %>
+<%     end -%>
+<% end -%>
 
   # Configuration options for specifying the Desktop Database. For more info,
   # see http://docs.djangoproject.com/en/1.4/ref/settings/#database-engine
@@ -361,8 +334,7 @@
     # Default email address to use for various automated notification from Hue
     ## default_from_email=hue@localhost
 
-
-<% if @kerberos_realm != "" %>
+<% if @kerberos_realm != "" -%>
   # Configuration options for Kerberos integration for secured Hadoop clusters
   # ------------------------------------------------------------------------
   [[kerberos]]
@@ -370,12 +342,11 @@
     # Path to Hue's Kerberos keytab file
     hue_keytab=/etc/hue.keytab
     # Kerberos principal name for Hue
-    hue_principal=hue/<%= @fqdn %>
+    hue_principal=hue/<%= @fqdn %>@<%= @kerberos_realm %>
     # Path to kinit
-    kinit_path=<%= (@operatingsystem == 'ubuntu') ? '/usr/bin' : '/usr/kerberos/bin' %>/kinit
-<% end %>
-
+    kinit_path=<%= (@operatingsystem == 'ubuntu' || @operatingsystem == 'Debian') ? '/usr/bin'
: '/usr/kerberos/bin' %>/kinit
 
+<% end -%>
   # Configuration options for using OAuthBackend (core) login
   # ------------------------------------------------------------------------
   [[oauth]]
@@ -616,25 +587,30 @@
   # Configuration for YARN (MR2)
   # ------------------------------------------------------------------------
   [[yarn_clusters]]
-
-    [[[default]]]
+<% resourcemanager_hosts = Array(@rm_host) -%>
+<% resourcemanager_hosts.each do |host| -%>
+  [[[<%= host %>]]]
       # Enter the host on which you are running the ResourceManager
-      resourcemanager_host=<%= @rm_host %>
+      resourcemanager_host=<%= host %>
 
       # The port where the ResourceManager IPC listens on
-      resourcemanager_port=<%= @rm_port %>
+      # resourcemanager_port=<% rm_port %>
 
       # Whether to submit jobs to this cluster
       submit_to=True
-
+<%     if @rm_logical_name -%>
       # Resource Manager logical name (required for HA)
-      ## logical_name=
+      logical_name=<%= @rm_logical_name %>
+<%     else -%>
+      # Resource Manager logical name (required for HA)
+      # logical_name=
+<%     end -%>
 
       # Change this if your YARN cluster is Kerberos-secured
       security_enabled=<%= if (@kerberos_realm != "") ; "true" else "false" end %>
 
       # URL of the ResourceManager API
-      resourcemanager_api_url=<%= @rm_url %>
+      resourcemanager_api_url=http://<%= host %>:<%= @rm_api_port %>
 
       # URL of the ProxyServer API
       proxy_api_url=<%= @rm_proxy_url %>
@@ -642,43 +618,12 @@
       # URL of the HistoryServer API
       history_server_api_url=<%= @history_server_url %>
 
-    # HA support by specifying multiple clusters
-    # e.g.
-
-    # [[[ha]]]
-      # Resource Manager logical name (required for HA)
-      # logical_name=my-rm-name
+      # URL of the NodeManager API
+      node_manager_api_url=http://localhost:8042
+<% end -%>
 
   # Configuration for MapReduce (MR1)
   # ------------------------------------------------------------------------
-  [[mapred_clusters]]
-
-    [[[default]]]
-      # Enter the host on which you are running the Hadoop JobTracker
-      ## jobtracker_host=localhost
-
-      # The port where the JobTracker IPC listens on
-      ## jobtracker_port=8021
-
-      # JobTracker logical name for HA
-      ## logical_name=
-
-      # Thrift plug-in port for the JobTracker
-      ## thrift_port=9290
-
-      # Whether to submit jobs to this cluster
-      submit_to=False
-
-      # Change this if your MapReduce cluster is Kerberos-secured
-      ## security_enabled=false
-
-    # HA support by specifying multiple clusters
-    # e.g.
-
-    # [[[ha]]]
-      # Enter the logical name of the JobTrackers
-      # logical_name=my-jt-name
-
 
 ###########################################################################
 # Settings to configure the Filebrowser app
@@ -694,10 +639,9 @@
 
 [liboozie]
   # The URL where the Oozie service runs on. This is required in order for
-  # users to submit jobs. Empty value disables the config check.
+  # users to submit jobs.
   oozie_url=<%= @oozie_url %>
 
-  # Requires FQDN in oozie_url if enabled
   security_enabled=<%= if (@kerberos_realm != "") ; "true" else "false" end %>
 
   # Location on HDFS where the workflows/coordinator are deployed when submitted.
@@ -716,10 +660,10 @@
   ## sample_data_dir=...thirdparty/sample_data
 
   # Location on HDFS where the oozie examples and workflows are stored.
-  remote_data_dir=/user/hue/oozie/workspaces
+  ## remote_data_dir=/user/hue/oozie/workspaces
 
   # Maximum of Oozie workflows or coodinators to retrieve in one API call.
-  oozie_jobs_count=100
+  ## oozie_jobs_count=100
 
   # Use Cron format for defining the frequency of a Coordinator instead of the old frequency
number/unit.
   ## enable_cron_scheduling=true
@@ -731,19 +675,24 @@
 
 [beeswax]
 
+<% if @hive_host -%>
   # Host where HiveServer2 is running.
   # If Kerberos security is enabled, use fully-qualified domain name (FQDN).
-  ## hive_server_host=localhost
+  hive_server_host=<%= @hive_host %>
 
   # Port where HiveServer2 Thrift server runs on.
-  ## hive_server_port=10000
+  hive_server_port=<%= @hive_port %>
 
   # Hive configuration directory, where hive-site.xml is located
-  ## hive_conf_dir=/etc/hive/conf
+  hive_conf_dir=/etc/hive/conf
 
   # Timeout in seconds for thrift calls to Hive service
   ## server_conn_timeout=120
 
+  # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs.
+  # If false, Hue will use the FetchResults() thrift call instead.
+  use_get_log_api=false
+
   # Set a LIMIT clause when browsing a partitioned table.
   # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
   ## browse_partitioned_table_limit=250
@@ -775,7 +724,7 @@
 
     # Choose whether Hue should validate certificates received from the server.
     ## validate=true
-
+<% end -%>
 
 ###########################################################################
 # Settings to configure Pig
@@ -797,8 +746,9 @@
   # For autocompletion, fill out the librdbms section.
 
   # Sqoop server URL
+<% if @sqoop_url != "" -%>
   server_url=<%= @sqoop_url %>
-
+<% end -%>
 
 ###########################################################################
 # Settings to configure Proxy
@@ -856,8 +806,11 @@
 [hbase]
   # Comma-separated list of HBase Thrift servers for clusters in the format of '(name|host:port)'.
   # Use full hostname with security.
+<% if @hbase_thrift_url != "" -%>
   hbase_clusters=(Bigtop|<%= @hbase_thrift_url %>)
-
+<% else -%>
+  ## hbase_clusters=(Cluster|localhost:9090)
+<% end -%>
   # HBase configuration directory, where hbase-site.xml is located.
   ## hbase_conf_dir=/etc/hbase/conf
 
@@ -877,13 +830,14 @@
 [search]
 
   # URL of the Solr Server
+<% if @solr_url != "" -%>
   solr_url=<%= @solr_url %>
-
+<% end -%>
   # Requires FQDN in solr_url if enabled
   ## security_enabled=false
 
   ## Query sent when no term is entered
-  empty_query=*:*
+  ## empty_query=*:*
 
 
 ###########################################################################
@@ -939,10 +893,10 @@
     [[[default]]]
       # Zookeeper ensemble. Comma separated list of Host/Port.
       # e.g. localhost:2181,localhost:2182,localhost:2183
-      ## host_ports=localhost:2181
+      host_ports=localhost:2181
 
       # The URL of the REST contrib service (required for znode browsing)
-      ## rest_url=http://localhost:9998
+      rest_url=http://localhost:9998
 
 
 ###########################################################################


Mime
View raw message