bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ofle...@apache.org
Subject [1/2] bigtop git commit: BIGTOP-2293. Add puppet recipes for qfs components
Date Fri, 25 Mar 2016 17:48:40 GMT
Repository: bigtop
Updated Branches:
  refs/heads/master 39c1c879c -> c1af7dc91


http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1af7dc9/bigtop-deploy/puppet/modules/qfs/templates/QfsClient.prp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/qfs/templates/QfsClient.prp b/bigtop-deploy/puppet/modules/qfs/templates/QfsClient.prp
new file mode 100644
index 0000000..94ae962
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/qfs/templates/QfsClient.prp
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The meta server configuration.
+
+# Where is the metaserver
+metaServer.name = <%= scope['qfs::common::metaserver_host'] %>
+metaServer.port = <%= scope['qfs::common::metaserver_port'] %>
+
+# -------------------- Client and meta server authentication. ------------------
+# By default QFS client and meta server authentication (client and chunk server
+# authentication as a consequence) is off.
+#
+# If any of the following meta authentication method is configured then QFS
+# client and the meta server perform mutual authentication.
+#
+# The QFS client configuration parameters can be specified also via environment
+# variables: QFS_CLIENT_CONFIG and QFS_CLIENT_CONFIG and
+# QFS_CLIENT_CONFIG_meta_server_ip_port. The later variable takes precedence.
+# The dots in the meta server ip (or host name) are replaced with _ (underscore)
+# symbols. The underscore symbol also used to separate meta server ip and port.
+# The later, longer form allows to use configuration specific to a
+# particular meta server, and mainly intended to be used with the QFS
+# delegation where both the delegation token and the key can be passed via
+# environment variables (see PSK authentication section below)),
+#
+# The two from environment vairable values are supported:
+# 1. FILE:configuration_file_name
+# 2. parameter_name1=parameter_value1 parameter_name2=parameter_value2...
+# The second space separated key value pairs can be used to pass delegation
+# token and the corresponding key. Both these must be obtained from the meta
+# server via "delegate" request. See qfs tool help.
+# For example:
+# QFS_CLIENT_CONFIG_127_0_0_1_20000='client.auth.psk.keyId=AAAB9dYIWfKBXhXCI1jJ9gAAU0XunwAAAACMoK0z30ztT5S7k9slRuRdzy9CXmi1
client.auth.psk.keyId=P+4XRIBLLBvkICXWO+1aXBPUTMghEakkTk1T+RVsifR9NQ71E32KVd27y+2DbyC2'
+# export QFS_CLIENT_CONFIG_127_0_0_1_20000
+
+
+# ================= X509 authentication ========================================
+#
+# QFS client's X509 certificate file in PEM format.
+# client.auth.X509.X509PemFile =
+
+# Password if X509 PEM file is encrypted.
+# client.auth.X509.X509Password =
+
+# QFS client's private key file.
+# client.auth.X509.PKeyPemFile =
+
+# Password if private key PEM file is encrypted.
+# client.auth.X509.PKeyPassword =
+
+# Certificate authorities file. Used for both meta server certificate
+# validation and to create certificate chain with QFS client's X509
+# certificate.
+# client.auth.X509.CAFile =
+
+# Certificate authorities directory can be used in addition to CAFile.
+# For more detailed information please see SSL_CTX_load_verify_locations manual
+# page. CAFile/CADir corresponds to CAfile/CApath in the man page.
+# client.auth.X509.CADir =
+
+# If set (the default) verify peer certificate, and declare error if peer, i.e.
+# meta server, does not preset "trusted" valid X509 certificate.
+# Default is on.
+# client.auth.X509.verifyPeer = 1
+
+# OpenSSL cipher configuration.
+# client.auth.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH
+
+# The long integer value passed to SSL_CTX_set_options() call.
+# See open ssl documentation for details.
+# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
+# client.auth.X509.options =
+
+# ================= Kerberos authentication ====================================
+#
+# Kerberos service principal: service/host@realm
+
+# Meta server's Kerberos principal [service/host@realm] service name part.
+# client.auth.krb5.service =
+
+# Meta server's Kerberos principal [service/host@realm] host name part.
+# client.auth.krb5.host =
+
+# Normally kinit is sufficient for the user authentication.
+# The following Kerberos parameters might be used in the case when another
+# "service" acts as QFS client.
+
+# Kerberos keytab file with the key(s) that corresponds to the QFS client's
+# principal, if used. Key table is typically used for service.
+# client.auth.krb5.keytab =
+
+# QFS client's kerberos principal. krb5_parse_name() used to convert the name
+# into the Kerberos 5 internal principal representation.
+# client.auth.krb5.clientName =
+
+# Force Kerberos client cache initialization during intialization.
+# Default is off.
+# client.auth.krb5.initClientCache = 0
+
+# OpenSSL cipher configuration for TLS-PSK authentication method. This method
+# is used with delegation and with Kerberos authentication.
+# client.auth.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH
+
+# The long integer value passed to SSL_CTX_set_options() call.
+# See open ssl documentation for details.
+# Default is the integer value that corresponds to the logical OR of
+# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
+# metaServer.clientAuthentication.psk.options =
+
+# ================= PSK / delegation authentication ============================
+#
+# Both delegation token and delegation key are expected to be valid base 64
+# encoded binary blobs -- the exact string representation returned by the
+# delegation request.
+
+# QFS client delegation token, The token must be obtained via delegation request
+# the meta server. Both the token and the corresponding key must be specified.
+# client.auth.psk.keyId =
+
+# QFS client delegation key, The key must be obtained via delegation request to
+# the meta server.
+# client.auth.psk.key =
+
+#-------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1af7dc9/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs b/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs
new file mode 100644
index 0000000..7041daf
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/qfs/templates/hadoop-qfs
@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A simple wrapper script around hadoop which enables the use of qfs easily.
+# Using the hadoop command with qfs requires users to specify overrides for
+# various parameters (e.g. fs.defaultFS) which can become cumbersome to do on
+# the command line every time. The goal here is to provide a clean user
+# experience in using hadoop and qfs.
+
+export JAVA_LIBRARY_PATH=/usr/lib/qfs
+
+QFS_OPTS=" -Dfs.qfs.impl=com.quantcast.qfs.hadoop.QuantcastFileSystem"
+QFS_OPTS="$QFS_OPTS -Dfs.defaultFS=qfs://<%= scope['qfs::common::metaserver_host'] %>:<%=
scope['qfs::common::metaserver_client_port'] %>"
+QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerHost=<%= scope['qfs::common::metaserver_host']
%>"
+QFS_OPTS="$QFS_OPTS -Dfs.qfs.metaServerPort=<%= scope['qfs::common::metaserver_port']
%>"
+
+usage() {
+  echo "
+usage: $0 <options>
+  $0 is a simple wrapper around the standard hadoop command that allows for an
+  easy interface with qfs. A subset of the standard hadoop commands are
+  supported (where it makes sense). For all other commands, use the hadoop
+  command directly.
+
+  Commands:
+     fs                             query the qfs filesystem
+     jar <jar> <main class>         submit a job to hadoop that uses qfs
+                                        Note: the main class is required
+     distcp <srcurl> <desturl>      copy file or directories recursively
+                                        Note: supports copy into qfs from hdfs
+  "
+  exit 1
+}
+
+if [ $# == 0 ]; then
+    usage
+fi
+
+COMMAND=$1
+shift
+
+case $COMMAND in
+    # usage flags
+    --help|-h)
+        usage
+        ;;
+    fs|distcp)
+        hadoop $COMMAND $QFS_OPTS "$@"
+        ;;
+    jar)
+        if [ $# -lt 2 ]; then
+            echo "$COMMAND: jar file and main class are required"
+            usage
+        fi
+
+        JAR_FILE=$1
+        MAIN_CLASS=$2
+        shift 2
+        hadoop $COMMAND $JAR_FILE $MAIN_CLASS $QFS_OPTS "$@"
+        ;;
+    *)
+        echo "$COMMAND: unsupported -- try using the hadoop command directly"
+        usage
+        ;;
+esac


Mime
View raw message