Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 27F39E311 for ; Fri, 1 Mar 2013 19:37:29 +0000 (UTC) Received: (qmail 74238 invoked by uid 500); 1 Mar 2013 19:37:28 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 74190 invoked by uid 500); 1 Mar 2013 19:37:28 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 74181 invoked by uid 99); 1 Mar 2013 19:37:28 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Mar 2013 19:37:28 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Mar 2013 19:37:26 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 1464B2388993; Fri, 1 Mar 2013 19:37:07 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1451695 - in /hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common: ./ src/ src/main/bin/ src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/io/ src/main/java/org/apache... Date: Fri, 01 Mar 2013 19:37:06 -0000 To: common-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130301193707.1464B2388993@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: suresh Date: Fri Mar 1 19:37:03 2013 New Revision: 1451695 URL: http://svn.apache.org/r1451695 Log: Merge trunk to branch-trunk-win Added: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java - copied unchanged from r1451693, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java - copied unchanged from r1451693, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed) hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/ (props changed) hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/ (props changed) hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/ (props changed) hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt Fri Mar 1 19:37:03 2013 @@ -355,6 +355,11 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9117. replace protoc ant plugin exec with a maven plugin. (tucu) + HADOOP-9279. Document the need to build hadoop-maven-plugins for + eclipse and separate project builds. (Tsuyoshi Ozawa via suresh) + + HADOOP-9334. Upgrade netty version. (Nicolas Liochon via suresh) + OPTIMIZATIONS BUG FIXES @@ -376,6 +381,28 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9304. remove addition of avro genreated-sources dirs to build. (tucu) + HADOOP-9267. hadoop -help, -h, --help should show usage instructions. + (Andrew Wang via atm) + + HADOOP-8569. CMakeLists.txt: define _GNU_SOURCE and _LARGEFILE_SOURCE. + (Colin Patrick McCabe via atm) + + HADOOP-9323. Fix typos in API documentation. (suresh) + + HADOOP-7487. DF should throw a more reasonable exception when mount cannot + be determined. (Andrew Wang via atm) + + HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern. + (Arpit Gupta via suresh) + + HADOOP-9342. Remove jline from distribution. (thw via tucu) + + HADOOP-9230. TestUniformSizeInputFormat fails intermittently. + (kkambatl via tucu) + + HADOOP-9349. Confusing output when running hadoop version from one hadoop + installation when HADOOP_HOME points to another. (sandyr via tucu) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES @@ -1351,7 +1378,10 @@ Release 0.23.7 - UNRELEASED permissions (Ivan A. Veselovsky via bobby) HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure - (Ivan A. Veselovsky via bobby) + (Ivan A. Veselovsky via bobby) + + HADOOP-9336. Allow UGI of current connection to be queried. (Daryn Sharp + via kihwal) OPTIMIZATIONS @@ -1363,6 +1393,9 @@ Release 0.23.7 - UNRELEASED HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage option (Andy Isaacson via tgraves) + HADOOP-9339. IPC.Server incorrectly sets UGI auth type (Daryn Sharp via + kihwal) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1448457-1451693 Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/pom.xml Fri Mar 1 19:37:03 2013 @@ -220,6 +220,10 @@ 3.4.2 + jline + jline + + junit junit Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/CMakeLists.txt Fri Mar 1 19:37:03 2013 @@ -90,7 +90,8 @@ find_package(ZLIB REQUIRED) SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64") set(D main/native/src/org/apache/hadoop) set(T main/native/src/test/org/apache/hadoop) Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/bin/hadoop Fri Mar 1 19:37:03 2013 @@ -50,6 +50,12 @@ fi COMMAND=$1 case $COMMAND in + # usage flags + --help|-help|-h) + print_usage + exit + ;; + #hdfs commands namenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|fetchdt|oiv|dfsgroups) echo "DEPRECATED: Use of this script to execute hdfs command is deprecated." 1>&2 Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/docs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:r1448457-1451693 Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1448457-1451693 Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java Fri Mar 1 19:37:03 2013 @@ -21,8 +21,6 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.Arrays; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -32,7 +30,7 @@ import org.apache.hadoop.util.PureJavaCr /**************************************************************** * Abstract Checksumed FileSystem. - * It provide a basice implementation of a Checksumed FileSystem, + * It provide a basic implementation of a Checksumed FileSystem, * which creates a checksum file for each raw file. * It generates & verifies checksums at the client side. * @@ -118,9 +116,6 @@ public abstract class ChecksumFileSystem * It verifies that data matches checksums. *******************************************************/ private static class ChecksumFSInputChecker extends FSInputChecker { - public static final Log LOG - = LogFactory.getLog(FSInputChecker.class); - private ChecksumFileSystem fs; private FSDataInputStream datas; private FSDataInputStream sums; @@ -376,19 +371,6 @@ public abstract class ChecksumFileSystem public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, - boolean overwrite, - short replication, - long blockSize, - Configuration conf) - throws IOException { - this(fs, file, overwrite, - conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, - LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT), - replication, blockSize, null); - } - - public ChecksumFSOutputSummer(ChecksumFileSystem fs, - Path file, boolean overwrite, int bufferSize, short replication, Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java Fri Mar 1 19:37:03 2013 @@ -17,19 +17,22 @@ */ package org.apache.hadoop.fs; +import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; -import java.io.BufferedReader; - +import java.util.ArrayList; import java.util.EnumSet; +import java.util.NoSuchElementException; import java.util.StringTokenizer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.util.Shell; +import com.google.common.annotations.VisibleForTesting; + /** Filesystem disk space usage statistics. * Uses the unix 'df' program to get mount points, and java.io.File for * space utilization. Tested on Linux, FreeBSD, Cygwin. */ @@ -44,6 +47,8 @@ public class DF extends Shell { private final File dirFile; private String filesystem; private String mount; + + private ArrayList output; enum OSType { OS_TYPE_UNIX("UNIX"), @@ -84,6 +89,7 @@ public class DF extends Shell { super(dfInterval); this.dirPath = path.getCanonicalPath(); this.dirFile = new File(this.dirPath); + this.output = new ArrayList(); } protected OSType getOSType() { @@ -127,7 +133,21 @@ public class DF extends Shell { /** @return the filesystem mount point for the indicated volume */ public String getMount() throws IOException { + // Abort early if specified path does not exist + if (!dirFile.exists()) { + throw new FileNotFoundException("Specified path " + dirFile.getPath() + + "does not exist"); + } run(); + // Skip parsing if df was not successful + if (getExitCode() != 0) { + StringBuffer sb = new StringBuffer("df could not be run successfully: "); + for (String line: output) { + sb.append(line); + } + throw new IOException(sb.toString()); + } + parseOutput(); return mount; } @@ -164,46 +184,71 @@ public class DF extends Shell { @Override protected void parseExecResult(BufferedReader lines) throws IOException { - lines.readLine(); // skip headings - + output.clear(); String line = lines.readLine(); - if (line == null) { - throw new IOException( "Expecting a line not the end of stream" ); + while (line != null) { + output.add(line); + line = lines.readLine(); + } + } + + @VisibleForTesting + protected void parseOutput() throws IOException { + if (output.size() < 2) { + StringBuffer sb = new StringBuffer("Fewer lines of output than expected"); + if (output.size() > 0) { + sb.append(": " + output.get(0)); + } + throw new IOException(sb.toString()); } + + String line = output.get(1); StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%"); - this.filesystem = tokens.nextToken(); + try { + this.filesystem = tokens.nextToken(); + } catch (NoSuchElementException e) { + throw new IOException("Unexpected empty line"); + } if (!tokens.hasMoreTokens()) { // for long filesystem name - line = lines.readLine(); - if (line == null) { - throw new IOException( "Expecting a line not the end of stream" ); + if (output.size() > 2) { + line = output.get(2); + } else { + throw new IOException("Expecting additional output after line: " + + line); } tokens = new StringTokenizer(line, " \t\n\r\f%"); } - switch(getOSType()) { - case OS_TYPE_AIX: - Long.parseLong(tokens.nextToken()); // capacity - Long.parseLong(tokens.nextToken()); // available - Integer.parseInt(tokens.nextToken()); // pct used - tokens.nextToken(); - tokens.nextToken(); - this.mount = tokens.nextToken(); - break; - - case OS_TYPE_WIN: - case OS_TYPE_SOLARIS: - case OS_TYPE_MAC: - case OS_TYPE_UNIX: - default: - Long.parseLong(tokens.nextToken()); // capacity - Long.parseLong(tokens.nextToken()); // used - Long.parseLong(tokens.nextToken()); // available - Integer.parseInt(tokens.nextToken()); // pct used - this.mount = tokens.nextToken(); - break; - } + try { + switch(getOSType()) { + case OS_TYPE_AIX: + Long.parseLong(tokens.nextToken()); // capacity + Long.parseLong(tokens.nextToken()); // available + Integer.parseInt(tokens.nextToken()); // pct used + tokens.nextToken(); + tokens.nextToken(); + this.mount = tokens.nextToken(); + break; + + case OS_TYPE_WIN: + case OS_TYPE_SOLARIS: + case OS_TYPE_MAC: + case OS_TYPE_UNIX: + default: + Long.parseLong(tokens.nextToken()); // capacity + Long.parseLong(tokens.nextToken()); // used + Long.parseLong(tokens.nextToken()); // available + Integer.parseInt(tokens.nextToken()); // pct used + this.mount = tokens.nextToken(); + break; + } + } catch (NoSuchElementException e) { + throw new IOException("Could not parse line: " + line); + } catch (NumberFormatException e) { + throw new IOException("Could not parse line: " + line); + } } public static void main(String[] args) throws Exception { Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Fri Mar 1 19:37:03 2013 @@ -1326,7 +1326,7 @@ public final class FileContext { * * 2. Partially qualified URIs (eg scheme but no host) * - * fs:///A/B/file Resolved according to the target file sytem. Eg resolving + * fs:///A/B/file Resolved according to the target file system. Eg resolving * a symlink to hdfs:///A results in an exception because * HDFS URIs must be fully qualified, while a symlink to * file:///A will not since Hadoop's local file systems Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Fri Mar 1 19:37:03 2013 @@ -1864,7 +1864,7 @@ public abstract class FileSystem extends * * Some file systems like LocalFileSystem have an initial workingDir * that we use as the starting workingDir. For other file systems - * like HDFS there is no built in notion of an inital workingDir. + * like HDFS there is no built in notion of an initial workingDir. * * @return if there is built in notion of workingDir then it * is returned; else a null is returned. Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java Fri Mar 1 19:37:03 2013 @@ -43,7 +43,7 @@ public interface PositionedReadable { throws IOException; /** - * Read number of bytes equalt to the length of the buffer, from a given + * Read number of bytes equal to the length of the buffer, from a given * position within a file. This does not * change the current offset of a file, and is thread-safe. */ Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java Fri Mar 1 19:37:03 2013 @@ -79,19 +79,17 @@ public abstract class TrashPolicy extend /** * Get an instance of the configured TrashPolicy based on the value - * of the configuration paramater fs.trash.classname. + * of the configuration parameter fs.trash.classname. * * @param conf the configuration to be used * @param fs the file system to be used * @param home the home directory * @return an instance of TrashPolicy */ - public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) - throws IOException { - Class trashClass = conf.getClass("fs.trash.classname", - TrashPolicyDefault.class, - TrashPolicy.class); - TrashPolicy trash = (TrashPolicy) ReflectionUtils.newInstance(trashClass, conf); + public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) { + Class trashClass = conf.getClass( + "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class); + TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf); trash.initialize(conf, fs, home); // initialize TrashPolicy return trash; } Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java Fri Mar 1 19:37:03 2013 @@ -27,7 +27,7 @@ import org.apache.hadoop.classification. /** * A byte sequence that is usable as a key or value. - * It is resizable and distinguishes between the size of the seqeunce and + * It is resizable and distinguishes between the size of the sequence and * the current capacity. The hash function is the front of the md5 of the * buffer. The sort order is the same as memcmp. */ Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java Fri Mar 1 19:37:03 2013 @@ -128,7 +128,7 @@ public class Text extends BinaryComparab /** * Returns the Unicode Scalar Value (32-bit integer value) * for the character at position. Note that this - * method avoids using the converter or doing String instatiation + * method avoids using the converter or doing String instantiation * @return the Unicode scalar value at position or -1 * if the position is invalid or points to a * trailing byte @@ -527,7 +527,7 @@ public class Text extends BinaryComparab int length = 0; int state = LEAD_BYTE; while (count < start+len) { - int aByte = ((int) utf8[count] & 0xFF); + int aByte = utf8[count] & 0xFF; switch (state) { case LEAD_BYTE: Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Fri Mar 1 19:37:03 2013 @@ -313,6 +313,14 @@ public abstract class Server { return (addr == null) ? null : addr.getHostAddress(); } + /** Returns the RPC remote user when invoked inside an RPC. Note this + * may be different than the current user if called within another doAs + * @return connection's UGI or null if not an RPC + */ + public static UserGroupInformation getRemoteUser() { + Call call = CurCall.get(); + return (call != null) ? call.connection.user : null; + } /** Return true if the invocation was through an RPC. */ @@ -1553,9 +1561,6 @@ public abstract class Server { UserGroupInformation protocolUser = ProtoUtil.getUgi(connectionContext); if (saslServer == null) { user = protocolUser; - if (user != null) { - user.setAuthenticationMethod(AuthMethod.SIMPLE); - } } else { // user is authenticated user.setAuthenticationMethod(authMethod); Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java Fri Mar 1 19:37:03 2013 @@ -192,7 +192,7 @@ public class Buffer implements Comparabl int hash = 1; byte[] b = this.get(); for (int i = 0; i < count; i++) - hash = (31 * hash) + (int)b[i]; + hash = (31 * hash) + b[i]; return hash; } Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java Fri Mar 1 19:37:03 2013 @@ -26,7 +26,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; /** - * Interface that alll the serializers have to implement. + * Interface that all the serializers have to implement. * * @deprecated Replaced by Avro. */ Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java Fri Mar 1 19:37:03 2013 @@ -30,6 +30,7 @@ import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.ServiceLoader; import java.util.Set; @@ -219,7 +220,7 @@ public class SecurityUtil { if (fqdn == null || fqdn.isEmpty() || fqdn.equals("0.0.0.0")) { fqdn = getLocalHostName(); } - return components[0] + "/" + fqdn.toLowerCase() + "@" + components[2]; + return components[0] + "/" + fqdn.toLowerCase(Locale.US) + "@" + components[2]; } static String getLocalHostName() throws UnknownHostException { Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java Fri Mar 1 19:37:03 2013 @@ -18,6 +18,11 @@ package org.apache.hadoop.util; +import java.io.IOException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Enumeration; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -155,5 +160,7 @@ public class VersionInfo { System.out.println("Subversion " + getUrl() + " -r " + getRevision()); System.out.println("Compiled by " + getUser() + " on " + getDate()); System.out.println("From source with checksum " + getSrcChecksum()); + System.out.println("This command was run using " + + ClassUtil.findContainingJar(VersionInfo.class)); } } Propchange: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/core/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:r1448457-1451693 Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java Fri Mar 1 19:37:03 2013 @@ -17,15 +17,22 @@ */ package org.apache.hadoop.fs; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.StringReader; import java.util.EnumSet; +import java.util.Random; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; +import org.junit.Test; -public class TestDFVariations extends TestCase { +public class TestDFVariations { public static class XXDF extends DF { private final String osName; @@ -50,6 +57,7 @@ public class TestDFVariations extends Te } } + @Test(timeout=5000) public void testOSParsing() throws Exception { for (DF.OSType ost : EnumSet.allOf(DF.OSType.class)) { XXDF df = new XXDF(ost.getId()); @@ -58,6 +66,74 @@ public class TestDFVariations extends Te df.getMount()); } } - + + @Test(timeout=5000) + public void testDFInvalidPath() throws Exception { + // Generate a path that doesn't exist + Random random = new Random(0xDEADBEEFl); + File file = null; + byte[] bytes = new byte[64]; + while (file == null) { + random.nextBytes(bytes); + final String invalid = new String("/" + bytes); + final File invalidFile = new File(invalid); + if (!invalidFile.exists()) { + file = invalidFile; + } + } + DF df = new DF(file, 0l); + try { + df.getMount(); + } catch (FileNotFoundException e) { + // expected, since path does not exist + GenericTestUtils.assertExceptionContains(file.getName(), e); + } + } + + @Test(timeout=5000) + public void testDFMalformedOutput() throws Exception { + DF df = new DF(new File("/"), 0l); + BufferedReader reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on\n" + + "/dev/sda5 19222656 10597036 7649060 59% /")); + df.parseExecResult(reader); + df.parseOutput(); + + reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on")); + df.parseExecResult(reader); + try { + df.parseOutput(); + fail("Expected exception with missing line!"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains( + "Fewer lines of output than expected", e); + System.out.println(e.toString()); + } + + reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on\n" + + " ")); + df.parseExecResult(reader); + try { + df.parseOutput(); + fail("Expected exception with empty line!"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Unexpected empty line", e); + System.out.println(e.toString()); + } + + reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on\n" + + " 19222656 10597036 7649060 59% /")); + df.parseExecResult(reader); + try { + df.parseOutput(); + fail("Expected exception with missing field!"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Could not parse line: ", e); + System.out.println(e.toString()); + } + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java?rev=1451695&r1=1451694&r2=1451695&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java Fri Mar 1 19:37:03 2013 @@ -127,6 +127,7 @@ public class TestDoAsEffectiveUser { public static final long versionID = 1L; String aMethod() throws IOException; + String getServerRemoteUser() throws IOException; } public class TestImpl implements TestProtocol { @@ -137,6 +138,11 @@ public class TestDoAsEffectiveUser { } @Override + public String getServerRemoteUser() throws IOException { + return Server.getRemoteUser().toString(); + } + + @Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { return TestProtocol.versionID; @@ -149,7 +155,23 @@ public class TestDoAsEffectiveUser { } } - @Test + private void checkRemoteUgi(final Server server, + final UserGroupInformation ugi, final Configuration conf) + throws Exception { + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException { + proxy = RPC.getProxy( + TestProtocol.class, TestProtocol.versionID, + NetUtils.getConnectAddress(server), conf); + Assert.assertEquals(ugi.toString(), proxy.aMethod()); + Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser()); + return null; + } + }); + } + + @Test(timeout=4000) public void testRealUserSetup() throws IOException { final Configuration conf = new Configuration(); conf.setStrings(ProxyUsers @@ -163,24 +185,13 @@ public class TestDoAsEffectiveUser { try { server.start(); - final InetSocketAddress addr = NetUtils.getConnectAddress(server); - UserGroupInformation realUserUgi = UserGroupInformation .createRemoteUser(REAL_USER_NAME); + checkRemoteUgi(server, realUserUgi, conf); + UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting( PROXY_USER_NAME, realUserUgi, GROUP_NAMES); - String retVal = proxyUserUgi - .doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws IOException { - proxy = RPC.getProxy(TestProtocol.class, - TestProtocol.versionID, addr, conf); - String ret = proxy.aMethod(); - return ret; - } - }); - - Assert.assertEquals(PROXY_USER_NAME + " (auth:SIMPLE) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); + checkRemoteUgi(server, proxyUserUgi, conf); } catch (Exception e) { e.printStackTrace(); Assert.fail(); @@ -192,7 +203,7 @@ public class TestDoAsEffectiveUser { } } - @Test + @Test(timeout=4000) public void testRealUserAuthorizationSuccess() throws IOException { final Configuration conf = new Configuration(); configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME); @@ -206,25 +217,13 @@ public class TestDoAsEffectiveUser { try { server.start(); - final InetSocketAddress addr = NetUtils.getConnectAddress(server); - UserGroupInformation realUserUgi = UserGroupInformation .createRemoteUser(REAL_USER_NAME); + checkRemoteUgi(server, realUserUgi, conf); UserGroupInformation proxyUserUgi = UserGroupInformation .createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES); - String retVal = proxyUserUgi - .doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws IOException { - proxy = RPC.getProxy(TestProtocol.class, - TestProtocol.versionID, addr, conf); - String ret = proxy.aMethod(); - return ret; - } - }); - - Assert.assertEquals(PROXY_USER_NAME + " (auth:SIMPLE) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); + checkRemoteUgi(server, proxyUserUgi, conf); } catch (Exception e) { e.printStackTrace(); Assert.fail();