hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1208644 - in /hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common: CHANGES.txt src/main/docs/ src/main/java/ src/main/packages/hadoop-setup-conf.sh src/main/packages/templates/conf/hdfs-site.xml src/test/core/
Date Wed, 30 Nov 2011 18:27:19 GMT
Author: atm
Date: Wed Nov 30 18:27:04 2011
New Revision: 1208644

URL: http://svn.apache.org/viewvc?rev=1208644&view=rev
Log:
Merge trunk into HA branch.

Modified:
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt   (contents,
props changed)
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/  
(props changed)
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/  
(props changed)
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml
    hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/  
(props changed)

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1208644&r1=1208643&r2=1208644&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov
30 18:27:04 2011
@@ -126,6 +126,9 @@ Release 0.23.1 - Unreleased
     HADOOP-7424. Log an error if the topology script doesn't handle multiple args.
     (Uma Maheswara Rao G via eli)
 
+    HADOOP-7804. Enable hadoop config generator to set configurations to enable
+    short circuit read. (Arpit Gupta via jitendra)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -141,6 +144,9 @@ Release 0.23.1 - Unreleased
 
    HADOOP-7859. TestViewFsHdfs.testgetFileLinkStatus is failing an assert. (eli)
 
+   HADOOP-7864. Building mvn site with Maven < 3.0.2 causes OOM errors.
+   (Andrew Bayer via eli)
+
 Release 0.23.0 - 2011-11-01 
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,5 +1,5 @@
 /hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1208622
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
 /hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1208622
 /hadoop/core/branches/branch-0.19/src/docs:713112

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1208622
 /hadoop/core/branches/branch-0.19/core/src/java:713112
 /hadoop/core/trunk/src/core:776175-785643,785929-786278

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh?rev=1208644&r1=1208643&r2=1208644&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
Wed Nov 30 18:27:04 2011
@@ -67,6 +67,10 @@ usage: $0 <parameters>
                                                                      This value should be
<= mapred.cluster.max.map.memory.mb
      --mapreduce-reduce-memory-mb=memory                             Virtual memory, of a
single reduce slot for a job. Defaults to -1
                                                                      This value should be
<= mapred.cluster.max.reduce.memory.mb
+     --dfs-datanode-dir-perm=700                                     Set the permission for
the datanode data directories. Defaults to 700
+     --dfs-block-local-path-access-user=user                         User for which you want
to enable shortcircuit read.
+     --dfs-client-read-shortcircuit=true/false                       Enable shortcircuit
read for the client. Will default to true if the shortcircuit user is set.
+     --dfs-client-read-shortcircuit-skip-checksum=false/true         Disable checking of
checksum when shortcircuit read is taking place. Defaults to false.
   "
   exit 1
 }
@@ -124,7 +128,7 @@ function addPropertyToXMLConf
   local finalVal=$5
 
   #create the property text, make sure the / are escaped
-  propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>"
+  propText="<property>\n<name>$property<\/name>\n<value>$propValue<\/value>\n"
   #if description is not empty add it
   if [ ! -z $desc ]
   then
@@ -147,6 +151,28 @@ function addPropertyToXMLConf
 }
 
 ##########################################
+# Function to setup up the short circuit read settings
+#########################################
+function setupShortCircuitRead
+{
+  local conf_file="${HADOOP_CONF_DIR}/hdfs-site.xml"
+  #if the shortcircuit user is not set then return
+  if [ -z $DFS_BLOCK_LOCAL_PATH_ACCESS_USER ]
+  then
+    return
+  fi
+  
+  #set the defaults if values not present
+  DFS_CLIENT_READ_SHORTCIRCUIT=${DFS_CLIENT_READ_SHORTCIRCUIT:-false}
+  DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=${DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM:-false}
+
+  #add the user to the conf file
+  addPropertyToXMLConf "$conf_file" "dfs.block.local-path-access.user" "$DFS_BLOCK_LOCAL_PATH_ACCESS_USER"
+  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit" "$DFS_CLIENT_READ_SHORTCIRCUIT"
+  addPropertyToXMLConf "$conf_file" "dfs.client.read.shortcircuit.skip.checksum" "$DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM"
+}
+
+##########################################
 # Function to setup up the proxy user settings
 #########################################
 function setupProxyUsers
@@ -217,6 +243,10 @@ OPTS=$(getopt \
   -l 'mapreduce-jobtracker-maxreducememory-mb:' \
   -l 'mapreduce-map-memory-mb:' \
   -l 'mapreduce-reduce-memory-mb:' \
+  -l 'dfs-datanode-dir-perm:' \
+  -l 'dfs-block-local-path-access-user:' \
+  -l 'dfs-client-read-shortcircuit:' \
+  -l 'dfs-client-read-shortcircuit-skip-checksum:' \   
   -o 'h' \
   -- "$@") 
   
@@ -376,6 +406,22 @@ while true ; do
       MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
       AUTOMATED=1
       ;;
+    --dfs-datanode-dir-perm)
+      DFS_DATANODE_DIR_PERM=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --dfs-block-local-path-access-user)
+      DFS_BLOCK_LOCAL_PATH_ACCESS_USER=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --dfs-client-read-shortcircuit)
+      DFS_CLIENT_READ_SHORTCIRCUIT=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --dfs-client-read-shortcircuit-skip-checksum)
+      DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM=$2; shift 2
+      AUTOMATED=1
+      ;;
     --)
       shift ; break
       ;;
@@ -421,6 +467,8 @@ DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:
 KERBEROS_REALM=${KERBEROS_REALM:-KERBEROS.EXAMPLE.COM}
 SECURITY_TYPE=${SECURITY_TYPE:-simple}
 KINIT=${KINIT:-/usr/kerberos/bin/kinit}
+#deault the data dir perm to 700
+DFS_DATANODE_DIR_PERM=${DFS_DATANODE_DIR_PERM:-700}
 if [ "${SECURITY_TYPE}" = "kerberos" ]; then
   TASK_CONTROLLER="org.apache.hadoop.mapred.LinuxTaskController"
   HADOOP_DN_ADDR="0.0.0.0:1019"
@@ -561,7 +609,10 @@ if [ "${AUTOSETUP}" == "1" -o "${AUTOSET
 
   #setup up the proxy users
   setupProxyUsers
-  
+ 
+  #setup short circuit read
+  setupShortCircuitRead
+
   #set the owner of the hadoop dir to root
   chown root ${HADOOP_PREFIX}
   chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
@@ -611,6 +662,9 @@ else
   #setup up the proxy users
   setupProxyUsers
   
+  #setup short circuit read
+  setupShortCircuitRead
+
   chown root:${HADOOP_GROUP} ${HADOOP_CONF_DIR}/hadoop-env.sh
   chmod 755 ${HADOOP_CONF_DIR}/hadoop-env.sh
   #set taskcontroller

Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml?rev=1208644&r1=1208643&r2=1208644&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hdfs-site.xml
Wed Nov 30 18:27:04 2011
@@ -202,7 +202,7 @@
 
   <property>
     <name>dfs.datanode.data.dir.perm</name>
-    <value>700</value>
+    <value>${DFS_DATANODE_DIR_PERM}</value>
     <description>The permissions that should be there on dfs.data.dir
       directories. The datanode will not come up if the permissions are
       different on existing dfs.data.dir directories. If the directories

Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Nov 30 18:27:04 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1208001
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1208622
 /hadoop/core/branches/branch-0.19/core/src/test/core:713112
 /hadoop/core/trunk/src/test/core:776175-785643,785929-786278



Mime
View raw message