hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject svn commit: r1206079 - in /hadoop/common/branches/branch-0.20-security: CHANGES.txt src/packages/hadoop-setup-conf.sh src/packages/templates/conf/hdfs-site.xml
Date Fri, 25 Nov 2011 08:58:16 GMT
Author: jitendra
Date: Fri Nov 25 08:58:15 2011
New Revision: 1206079

URL: http://svn.apache.org/viewvc?rev=1206079&view=rev
Log:
HADOOP-7804. Enable hadoop config generator to set configurations to enable short circuit
read. Contributed by Arpit Gupta.

Modified:
    hadoop/common/branches/branch-0.20-security/CHANGES.txt
    hadoop/common/branches/branch-0.20-security/src/packages/hadoop-setup-conf.sh
    hadoop/common/branches/branch-0.20-security/src/packages/templates/conf/hdfs-site.xml

Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1206079&r1=1206078&r2=1206079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Fri Nov 25 08:58:15 2011
@@ -140,6 +140,9 @@ Release 0.20.205.1 - unreleased
 
     HADOOP-7810. Add tools jar to the end of classpath. (John George via omalley)
 
+    HADOOP-7804. Enable hadoop config generator to set configurations to enable 
+    short circuit read. (Arpit Gupta via jitendra)
+
   BUG FIXES
 
     HADOOP-7827. jsp pages missing DOCTYPE. (Dave Vronay via mattf)

Modified: hadoop/common/branches/branch-0.20-security/src/packages/hadoop-setup-conf.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/packages/hadoop-setup-conf.sh?rev=1206079&r1=1206078&r2=1206079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/packages/hadoop-setup-conf.sh (original)
+++ hadoop/common/branches/branch-0.20-security/src/packages/hadoop-setup-conf.sh Fri Nov
25 08:58:15 2011
@@ -65,6 +65,10 @@ usage: $0 <parameters>
                                                                      This value should be
<= mapred.cluster.max.map.memory.mb
      --mapred-job-reduce-memory-mb=memory                            Virtual memory, of a
single reduce slot for a job. Defaults to -1
                                                                      This value should be
<= mapred.cluster.max.reduce.memory.mb
+     --dfs-datanode-dir-perm=700                                     Set the permission for
the datanode data directories. Defaults to 700
+     --dfs-block-local-path-access-user=user                         User for which you want
to enable shortcircuit read.
+     --dfs-client-read-shortcircuit=true/false                       Enable shortcircuit
read for the client. Will default to true if the shortcircuit user is set.
+     --dfs-client-read-shortcircuit-skip-checksum=false/true         Disable checking of
checksum when shortcircuit read is taking place. Defaults to false.
   "
   exit 1
 }
@@ -122,7 +126,7 @@ function addPropertyToXMLConf
   local finalVal=$5
 
   #create the property text, make sure the / are escaped
-  propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>"
+  propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>\n"
   #if description is not empty add it
   if [ ! -z $desc ]
   then
@@ -145,6 +149,28 @@ function addPropertyToXMLConf
 }
 
 ##########################################
+# Function to setup up the short circuit read settings
+#########################################
+function setupShortCircuitRead
+{
+  local conf_file="${HADOOP_CONF_DIR}/hdfs-site.xml"
+  #if the shortcircuit user is not set then return
+  if [ -z $DFS_BLOCK_LOCAL_PATH_ACCESS_USER ]
+  then
+    return
+  fi
+  
+  #set the defaults if values not present
+  DFS_CLIENT_READ_SHORTCIRCUIT=${DFS_CLIENT_READ_SHORTCIRCUIT:-false}
+  DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=${DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM:-false}
+
+  #add the user to the conf file
+  addPropertyToXMLConf "$conf_file" "dfs.block.local-path-access.user" "$DFS_BLOCK_LOCAL_PATH_ACCESS_USER"
+  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit" "$DFS_CLIENT_READ_SHORTCIRCUIT"
+  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit.skip.checksum" "$DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM"
+}
+
+##########################################
 # Function to setup up the proxy user settings
 #########################################
 function setupProxyUsers
@@ -215,6 +241,10 @@ OPTS=$(getopt \
   -l 'mapred-cluster-max-reduce-memory-mb:' \
   -l 'mapred-job-map-memory-mb:' \
   -l 'mapred-job-reduce-memory-mb:' \
+  -l 'dfs-datanode-dir-perm:' \
+  -l 'dfs-block-local-path-access-user:' \
+  -l 'dfs-client-read-shortcircuit:' \
+  -l 'dfs-client-read-shortcircuit-skip-checksum:' \
   -o 'h' \
   -- "$@") 
   
@@ -374,6 +404,22 @@ while true ; do
       MAPRED_JOB_REDUCE_MEMORY_MB=$2; shift 2
       AUTOMATED=1
       ;;
+    --dfs-datanode-dir-perm)
+      DFS_DATANODE_DIR_PERM=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --dfs-block-local-path-access-user)
+      DFS_BLOCK_LOCAL_PATH_ACCESS_USER=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --dfs-client-read-shortcircuit)
+      DFS_CLIENT_READ_SHORTCIRCUIT=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --dfs-client-read-shortcircuit-skip-checksum)
+      DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=$2; shift 2
+      AUTOMATED=1
+      ;;
     --)
       shift ; break
       ;;
@@ -416,6 +462,8 @@ HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hd
 MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
 KERBEROS_REALM=${KERBEROS_REALM:-KERBEROS.EXAMPLE.COM}
 SECURITY_TYPE=${SECURITY_TYPE:-simple}
+#deault the data dir perm to 700
+DFS_DATANODE_DIR_PERM=${DFS_DATANODE_DIR_PERM:-700}
 KINIT=${KINIT:-/usr/kerberos/bin/kinit}
 if [ "${SECURITY_TYPE}" = "kerberos" ]; then
   TASK_CONTROLLER="org.apache.hadoop.mapred.LinuxTaskController"
@@ -555,6 +603,9 @@ if [ "${AUTOSETUP}" == "1" -o "${AUTOSET
   #setup up the proxy users
   setupProxyUsers
 
+  #setup short circuit read
+  setupShortCircuitRead
+
   #set the owner of the hadoop dir to root
   chown root ${HADOOP_PREFIX}
   chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
@@ -602,6 +653,9 @@ else
   #setup up the proxy users
   setupProxyUsers
 
+  #setup shortcircuit read
+  setupShortCircuitRead
+
   chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
   chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
   #set taskcontroller

Modified: hadoop/common/branches/branch-0.20-security/src/packages/templates/conf/hdfs-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/packages/templates/conf/hdfs-site.xml?rev=1206079&r1=1206078&r2=1206079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/packages/templates/conf/hdfs-site.xml
(original)
+++ hadoop/common/branches/branch-0.20-security/src/packages/templates/conf/hdfs-site.xml
Fri Nov 25 08:58:15 2011
@@ -181,7 +181,7 @@
 
   <property>
     <name>dfs.datanode.data.dir.perm</name>
-    <value>700</value>
+    <value>${DFS_DATANODE_DIR_PERM}</value>
     <description>The permissions that should be there on dfs.data.dir
       directories. The datanode will not come up if the permissions are
       different on existing dfs.data.dir directories. If the directories



Mime
View raw message