hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1170378 - in /hadoop/common/branches/HDFS-1623: ./ dev-support/smart-apply-patch.sh dev-support/test-patch.sh hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
Date Tue, 13 Sep 2011 22:49:37 GMT
Author: atm
Date: Tue Sep 13 22:49:27 2011
New Revision: 1170378

URL: http://svn.apache.org/viewvc?rev=1170378&view=rev
Log:
Merge trunk into HA branch

Modified:
    hadoop/common/branches/HDFS-1623/   (props changed)
    hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh
    hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh
    hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml

Propchange: hadoop/common/branches/HDFS-1623/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 13 22:49:27 2011
@@ -1 +1 @@
-/hadoop/common/trunk:1152502-1166484
+/hadoop/common/trunk:1152502-1170371

Modified: hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh (original)
+++ hadoop/common/branches/HDFS-1623/dev-support/smart-apply-patch.sh Tue Sep 13 22:49:27
2011
@@ -39,40 +39,68 @@ fi
 # Come up with a list of changed files into $TMP
 TMP=/tmp/tmp.paths.$$
 TOCLEAN="$TOCLEAN $TMP"
-grep '^+++\|^---' $PATCH_FILE | cut -c '5-' | grep -v /dev/null | sort | uniq > $TMP
 
-# Assume p0 to start
-PLEVEL=0
-
-# if all of the lines start with a/ or b/, then this is a git patch that
-# was generated without --no-prefix
-if ! grep -qv '^a/\|^b/' $TMP ; then
-  echo Looks like this is a git patch. Stripping a/ and b/ prefixes
-  echo and incrementing PLEVEL
-  PLEVEL=$[$PLEVEL + 1]
-  sed -i -e 's,^[ab]/,,' $TMP
-fi
-
-PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
-
-# if we are at the project root then nothing more to do
-if [[ -d hadoop-common-project ]]; then
-  echo Looks like this is being run at project root
-
-# if all of the lines start with hadoop-common/, hadoop-hdfs/, or hadoop-mapreduce/, this
is
-# relative to the hadoop root instead of the subproject root, so we need
-# to chop off another layer
-elif [[ "$PREFIX_DIRS" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$
]]; then
-
-  echo Looks like this is relative to project root. Increasing PLEVEL
-  PLEVEL=$[$PLEVEL + 1]
-
-elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project'
; then
-  echo Looks like this is a cross-subproject patch. Try applying from the project root
-  exit 1
+if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
+  PLEVEL=0
+  #if the patch applied at P0 there is the possability that all we are doing
+  # is adding new files and they would apply anywhere. So try to guess the
+  # correct place to put those files.
+
+  TMP2=/tmp/tmp.paths.2.$$
+  TOCLEAN="$TOCLEAN $TMP2"
+
+  grep '^patching file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
+
+  #first off check that all of the files do not exist
+  FOUND_ANY=0
+  for CHECK_FILE in $(cat $TMP2)
+  do
+    if [[ -f $CHECK_FILE ]]; then
+      FOUND_ANY=1
+    fi
+  done
+
+  if [[ "$FOUND_ANY" = "0" ]]; then
+    #all of the files are new files so we have to guess where the correct place to put it
is.
+
+    # if all of the lines start with a/ or b/, then this is a git patch that
+    # was generated without --no-prefix
+    if ! grep -qv '^a/\|^b/' $TMP2 ; then
+      echo Looks like this is a git patch. Stripping a/ and b/ prefixes
+      echo and incrementing PLEVEL
+      PLEVEL=$[$PLEVEL + 1]
+      sed -i -e 's,^[ab]/,,' $TMP2
+    fi
+
+    PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 | sort | uniq)
+
+    # if we are at the project root then nothing more to do
+    if [[ -d hadoop-common-project ]]; then
+      echo Looks like this is being run at project root
+
+    # if all of the lines start with hadoop-common/, hadoop-hdfs/, or hadoop-mapreduce/,
this is
+    # relative to the hadoop root instead of the subproject root, so we need
+    # to chop off another layer
+    elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$
]]; then
+
+      echo Looks like this is relative to project root. Increasing PLEVEL
+      PLEVEL=$[$PLEVEL + 1]
+
+    elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project'
; then
+      echo Looks like this is a cross-subproject patch. Try applying from the project root
+      cleanup 1
+    fi
+  fi
+elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+  PLEVEL=1
+elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+  PLEVEL=2
+else
+  echo "The patch does not appear to apply with p0 to p2";
+  cleanup 1;
 fi
 
 echo Going to apply patch with: $PATCH -p$PLEVEL
 $PATCH -p$PLEVEL -E < $PATCH_FILE
 
-cleanup 0
+cleanup $?

Modified: hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/HDFS-1623/dev-support/test-patch.sh Tue Sep 13 22:49:27 2011
@@ -64,6 +64,7 @@ printUsage() {
   echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment
variable)"
   echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment
variable)"
   echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
+  echo "--run-tests            Run all tests below the base directory"
   echo
   echo "Jenkins-only options:"
   echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
@@ -130,6 +131,9 @@ parseArgs() {
     --dirty-workspace)
       DIRTY_WORKSPACE=true
       ;;
+    --run-tests)
+      RUN_TESTS=true
+      ;;
     *)
       PATCH_OR_DEFECT=$i
       ;;
@@ -249,6 +253,18 @@ setup () {
   echo "======================================================================"
   echo ""
   echo ""
+  if [[ ! -d hadoop-common-project ]]; then
+    cd $bindir/..
+    echo "Compiling $(pwd)"
+    echo "$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1"
+    $MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1
+    if [[ $? != 0 ]] ; then
+      echo "Top-level trunk compilation is broken?"
+      cleanupAndExit 1
+    fi
+    cd -
+  fi
+  echo "Compiling $(pwd)"
   echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt
2>&1"
   $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt
2>&1
   if [[ $? != 0 ]] ; then
@@ -550,8 +566,10 @@ checkFindbugsWarnings () {
     $FINDBUGS_HOME/bin/convertXmlToText -html \
       $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
       $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
-    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
+    if [[ $newFindbugsWarnings > 0 ]] ; then
+      JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/$(basename $BASEDIR)/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
 $JIRA_COMMENT_FOOTER"
+    fi
   done
 
   ### if current warnings greater than OK_FINDBUGS_WARNINGS
@@ -580,26 +598,12 @@ runTests () {
   echo ""
   echo ""
   
-  failed_tests=""
-  modules=$(findModules)
-  for module in $modules;
-  do
-    pushd $module
-      echo "    Running tests in $module"
-      ### Kill any rogue build processes from the last attempt
-      $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs
-t -I {} /bin/kill -9 {} > /dev/null
-
-      echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
-      $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
-      if [[ $? != 0 ]] ; then
-        ### Find and format names of failed tests
-        module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error"
| sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
-        failed_tests="${failed_tests}
-${module_failed_tests}"
-      fi
-    popd
-  done
-  echo $failed_tests
+  echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
+  $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
+  if [[ $? != 0 ]] ; then
+    ### Find and format names of failed tests
+    failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" |
sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`
+  fi
   
   if [[ -n "$failed_tests" ]] ; then
   
@@ -616,36 +620,6 @@ $failed_tests"
 }
 
 ###############################################################################
-### Find the modules changed by the patch
-
-findModules () {
-  # Come up with a list of changed files into $TMP
-  TMP=/tmp/tmp.paths.$$
-  $GREP '^+++\|^---' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null | sort | uniq >
$TMP
-
-  # if all of the lines start with a/ or b/, then this is a git patch that
-  # was generated without --no-prefix
-  if ! $GREP -qv '^a/\|^b/' $TMP ; then
-    sed -i -e 's,^[ab]/,,' $TMP
-  fi
-
-  PREFIX_DIRS=$(cut -d '/' -f 1 $TMP | sort | uniq)
-
-  # if all of the lines start with hadoop-common-project/, hadoop-hdfs-project/, or hadoop-mapreduce-project/,
this is
-  # relative to the hadoop root instead of the subproject root
-  if [[ "$PREFIX_DIRS" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-mapreduce-project)$
]]; then
-    echo $PREFIX_DIRS
-    return 0
-  elif ! echo "$PREFIX_DIRS" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-mapreduce-project'
; then
-    echo $PREFIX_DIRS
-    return 0
-  fi
-  
-  # No modules found. Running from current directory.
-  echo .
-}
-
-###############################################################################
 ### Run the test-contrib target
 runContribTests () {
   echo ""
@@ -820,8 +794,8 @@ checkFindbugsWarnings
 (( RESULT = RESULT + $? ))
 checkReleaseAuditWarnings
 (( RESULT = RESULT + $? ))
-### Do not call these when run by a developer 
-if [[ $JENKINS == "true" ]] ; then
+### Run tests for Jenkins or if explictly asked for by a developer
+if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
   runTests
   (( RESULT = RESULT + $? ))
   runContribTests

Modified: hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml?rev=1170378&r1=1170377&r2=1170378&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
(original)
+++ hadoop/common/branches/HDFS-1623/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
Tue Sep 13 22:49:27 2011
@@ -79,6 +79,13 @@
       </includes>
     </fileSet>
     <fileSet>
+      <directory>${basedir}/src/main/packages/templates/conf</directory>
+      <outputDirectory>/share/hadoop/${hadoop.component}/templates/conf</outputDirectory>
+      <includes>
+        <include>*</include>
+      </includes>
+    </fileSet>
+    <fileSet>
       <directory>${project.build.directory}</directory>
       <outputDirectory>/share/hadoop/${hadoop.component}</outputDirectory>
       <includes>



Mime
View raw message