From common-commits-return-15531-apmail-hadoop-common-commits-archive=hadoop.apache.org@hadoop.apache.org Tue Oct 4 22:50:50 2011 Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 87D8776EC for ; Tue, 4 Oct 2011 22:50:50 +0000 (UTC) Received: (qmail 44337 invoked by uid 500); 4 Oct 2011 22:50:50 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 44313 invoked by uid 500); 4 Oct 2011 22:50:50 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 44306 invoked by uid 99); 4 Oct 2011 22:50:50 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Oct 2011 22:50:50 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 04 Oct 2011 22:50:48 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A0732238897D for ; Tue, 4 Oct 2011 22:50:28 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1179000 - in /hadoop/common/branches/branch-0.20-security-205: CHANGES.txt src/packages/hadoop-setup-conf.sh src/packages/templates/conf/hdfs-site.xml Date: Tue, 04 Oct 2011 22:50:28 -0000 To: common-commits@hadoop.apache.org From: eyang@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111004225028.A0732238897D@eris.apache.org> Author: eyang Date: Tue Oct 4 22:50:28 2011 New Revision: 1179000 URL: http://svn.apache.org/viewvc?rev=1179000&view=rev Log: HADOOP-7707. Added toggle for dfs.support.append, webhdfs and hadoop proxy user to setup config script. (Arpit Gupta via Eric Yang) Modified: hadoop/common/branches/branch-0.20-security-205/CHANGES.txt hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/hdfs-site.xml Modified: hadoop/common/branches/branch-0.20-security-205/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/CHANGES.txt?rev=1179000&r1=1178999&r2=1179000&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-205/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.20-security-205/CHANGES.txt Tue Oct 4 22:50:28 2011 @@ -356,6 +356,9 @@ Release 0.20.205.0 - 2011.09.28 HADOOP-7708. Fixed hadoop-setup-conf.sh to handle config file consistently. (Eric Yang) + HADOOP-7707. Added toggle for dfs.support.append, webhdfs and hadoop proxy + user to setup config script. (Arpit Gupta via Eric Yang) + Release 0.20.204.0 - 2011-8-25 NEW FEATURES Modified: hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh?rev=1179000&r1=1178999&r2=1179000&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh (original) +++ hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh Tue Oct 4 22:50:28 2011 @@ -51,6 +51,9 @@ usage: $0 --taskscheduler=org.apache.hadoop.mapred.JobQueueTaskScheduler Set task scheduler --datanodes=hostname1,hostname2,... SET the datanodes --tasktrackers=hostname1,hostname2,... SET the tasktrackers + --dfs-webhdfs-enabled=false|true Enable webhdfs + --dfs-support-append=false|true Enable append + --hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts' Setup proxy users for hadoop " exit 1 } @@ -90,6 +93,78 @@ template_generator() { done } +######################################### +# Function to modify a value of a field in an xml file +# Params: $1 is the file with full path; $2 is the property, $3 is the new value +######################################### +function addPropertyToXMLConf +{ + #read the file name with full path + local file=$1 + #get the property name + local property=$2 + #get what value should be set for that + local propValue=$3 + #get the description + local desc=$4 + #get the value for the final tag + local finalVal=$5 + + #create the property text, make sure the / are escaped + propText="\n$property<\/name>\n$propValue<\/value>" + #if description is not empty add it + if [ ! -z $desc ] + then + propText="${propText}$desc<\/description>\n" + fi + + #if final is not empty add it + if [ ! -z $finalVal ] + then + propText="${propText}final>$finalVal<\/final>\n" + fi + + #add the ending tag + propText="${propText}<\/property>\n" + + #add the property to the file + endText="<\/configuration>" + #add the text using sed at the end of the file + sed -i "s|$endText|$propText$endText|" $file +} + +########################################## +# Function to setup up the proxy user settings +######################################### +function setupProxyUsers +{ + #if hadoop proxy users are sent, setup hadoop proxy + if [ ! -z $HADOOP_PROXY_USERS ] + then + oldIFS=$IFS + IFS=';' + #process each proxy config + for proxy in $HADOOP_PROXY_USERS + do + #get the user, group and hosts information for each proxy + IFS=':' + arr=($proxy) + user="${arr[0]}" + groups="${arr[1]}" + hosts="${arr[2]}" + #determine the property names and values + proxy_groups_property="hadoop.proxyuser.${user}.groups" + proxy_groups_val="$groups" + addPropertyToXMLConf "${HADOOP_CONF_DIR}/hdfs-site.xml" "$proxy_groups_property" "$proxy_groups_val" + proxy_hosts_property="hadoop.proxyuser.${user}.hosts" + proxy_hosts_val="$hosts" + addPropertyToXMLConf "${HADOOP_CONF_DIR}/hdfs-site.xml" "$proxy_hosts_property" "$proxy_hosts_val" + IFS=';' + done + IFS=$oldIFS + fi +} + OPTS=$(getopt \ -n $0 \ -o '' \ @@ -118,6 +193,9 @@ OPTS=$(getopt \ -l 'kinit-location:' \ -l 'datanodes:' \ -l 'tasktrackers:' \ + -l 'dfs-webhdfs-enabled:' \ + -l 'hadoop-proxy-users:' \ + -l 'dfs-support-append:' \ -o 'h' \ -- "$@") @@ -237,6 +315,18 @@ while true ; do AUTOMATED=1 TASKTRACKERS=$(echo $TASKTRACKERS | tr ',' ' ') ;; + --dfs-webhdfs-enabled) + DFS_WEBHDFS_ENABLED=$2; shift 2 + AUTOMATED=1 + ;; + --hadoop-proxy-users) + HADOOP_PROXY_USERS=$2; shift 2 + AUTOMATED=1 + ;; + --dfs-support-append) + DFS_SUPPORT_APPEND=$2; shift 2 + AUTOMATED=1 + ;; --) shift ; break ;; @@ -406,6 +496,9 @@ if [ "${AUTOSETUP}" == "1" -o "${AUTOSET template_generator ${HADOOP_PREFIX}/share/hadoop/templates/conf/log4j.properties ${HADOOP_CONF_DIR}/log4j.properties template_generator ${HADOOP_PREFIX}/share/hadoop/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties + #setup up the proxy users + setupProxyUsers + #set the owner of the hadoop dir to root chown root ${HADOOP_PREFIX} chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh @@ -450,6 +543,9 @@ else template_generator ${HADOOP_PREFIX}/share/hadoop/templates/conf/log4j.properties ${HADOOP_CONF_DIR}/log4j.properties template_generator ${HADOOP_PREFIX}/share/hadoop/templates/conf/hadoop-metrics2.properties ${HADOOP_CONF_DIR}/hadoop-metrics2.properties + #setup up the proxy users + setupProxyUsers + chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh #set taskcontroller Modified: hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/hdfs-site.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/hdfs-site.xml?rev=1179000&r1=1178999&r2=1179000&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/hdfs-site.xml (original) +++ hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/hdfs-site.xml Tue Oct 4 22:50:28 2011 @@ -228,4 +228,14 @@ excluded. + + dfs.webhdfs.enabled + ${DFS_WEBHDFS_ENABLED} + Enable or disable webhdfs. Defaults to false + + + dfs.support.append + ${DFS_SUPPORT_APPEND} + Enable or disable append. Defaults to false +