hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject svn commit: r885142 [3/6] - in /hadoop/common/branches/HADOOP-6194: ./ .eclipse.templates/ bin/ ivy/ lib/jdiff/ src/ src/contrib/ src/contrib/ec2/ src/docs/ src/docs/src/documentation/ src/docs/src/documentation/content/xdocs/ src/docs/src/documentatio...
Date Sat, 28 Nov 2009 19:53:40 GMT
Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSDataOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSDataOutputStream.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSDataOutputStream.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSDataOutputStream.java Sat Nov 28 19:53:33 2009
@@ -91,10 +91,29 @@
     return wrappedStream;
   }
 
-  /** {@inheritDoc} */
+  @Override  // Syncable
+  @Deprecated
   public void sync() throws IOException {
     if (wrappedStream instanceof Syncable) {
       ((Syncable)wrappedStream).sync();
     }
   }
+  
+  @Override  // Syncable
+  public void hflush() throws IOException {
+    if (wrappedStream instanceof Syncable) {
+      ((Syncable)wrappedStream).hflush();
+    } else {
+      wrappedStream.flush();
+    }
+  }
+  
+  @Override  // Syncable
+  public void hsync() throws IOException {
+    if (wrappedStream instanceof Syncable) {
+      ((Syncable)wrappedStream).hsync();
+    } else {
+      wrappedStream.flush();
+    }
+  }
 }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSInputChecker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSInputChecker.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSInputChecker.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FSInputChecker.java Sat Nov 28 19:53:33 2009
@@ -296,12 +296,12 @@
   
   @Override
   public synchronized long getPos() throws IOException {
-    return chunkPos-(count-pos);
+    return chunkPos-Math.max(0L, count - pos);
   }
 
   @Override
   public synchronized int available() throws IOException {
-    return count-pos;
+    return Math.max(0, count - pos);
   }
   
   /**

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileStatus.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileStatus.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileStatus.java Sat Nov 28 19:53:33 2009
@@ -148,6 +148,10 @@
   public Path getPath() {
     return path;
   }
+  
+  public void setPath(final Path p) {
+    path = p;
+  }
 
   /* These are provided so that these values could be loaded lazily 
    * by a filesystem (e.g. local file system).

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileSystem.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileSystem.java Sat Nov 28 19:53:33 2009
@@ -23,7 +23,6 @@
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -42,7 +41,10 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Options.CreateOpts;
+import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.io.MultipleIOException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
@@ -67,7 +69,10 @@
  * implementation is DistributedFileSystem.
  *****************************************************************/
 public abstract class FileSystem extends Configured implements Closeable {
-  public static final String FS_DEFAULT_NAME_KEY = "fs.default.name";
+  public static final String FS_DEFAULT_NAME_KEY = 
+                   CommonConfigurationKeys.FS_DEFAULT_NAME_KEY;
+  public static final String DEFAULT_FS = 
+                   CommonConfigurationKeys.FS_DEFAULT_NAME_DEFAULT;
 
   public static final Log LOG = LogFactory.getLog(FileSystem.class);
 
@@ -103,7 +108,7 @@
    * @return the uri of the default filesystem
    */
   public static URI getDefaultUri(Configuration conf) {
-    return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, "file:///")));
+    return URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
   }
 
   /** Set the default filesystem URI in a configuration.
@@ -181,6 +186,11 @@
         return get(defaultUri, conf);              // return default
       }
     }
+    
+    String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
+    if (conf.getBoolean(disableCacheName, false)) {
+      return createFileSystem(uri, conf);
+    }
 
     return CACHE.get(uri, conf);
   }
@@ -239,7 +249,7 @@
   /** Make sure that a path specifies a FileSystem. */
   public Path makeQualified(Path path) {
     checkPath(path);
-    return path.makeQualified(this);
+    return path.makeQualified(this.getUri(), this.getWorkingDirectory());
   }
     
   /** create a file with the provided permission
@@ -358,8 +368,42 @@
     String[] host = { "localhost" };
     return new BlockLocation[] { new BlockLocation(name, host, 0, file.getLen()) };
   }
+ 
+
+  /**
+   * Return an array containing hostnames, offset and size of 
+   * portions of the given file.  For a nonexistent 
+   * file or regions, null will be returned.
+   *
+   * This call is most helpful with DFS, where it returns 
+   * hostnames of machines that contain the given file.
+   *
+   * The FileSystem will simply return an elt containing 'localhost'.
+   */
+  public BlockLocation[] getFileBlockLocations(Path p, 
+      long start, long len) throws IOException {
+    if (p == null) {
+      throw new NullPointerException();
+    }
+    FileStatus file = getFileStatus(p);
+    return getFileBlockLocations(file, start, len);
+  }
   
   /**
+   * Return a set of server default configuration values
+   * @return server default configuration values
+   * @throws IOException
+   */
+  public FsServerDefaults getServerDefaults() throws IOException {
+    Configuration conf = getConf();
+    return new FsServerDefaults(getDefaultBlockSize(), 
+        conf.getInt("io.bytes.per.checksum", 512), 
+        64 * 1024, 
+        getDefaultReplication(), 
+        conf.getInt("io.file.buffer.size", 4096));
+  }
+
+  /**
    * Opens an FSDataInputStream at the indicated Path.
    * @param f the file name to open
    * @param bufferSize the size of the buffer to be used.
@@ -536,7 +580,7 @@
    * Opens an FSDataOutputStream at the indicated Path with write-progress
    * reporting.
    * @param f the file name to open.
-   * @param permission
+   * @param permission - applied against umask
    * @param flag determines the semantic of this create.
    * @param bufferSize the size of the buffer to be used.
    * @param replication required block replication for the file.
@@ -550,6 +594,183 @@
       EnumSet<CreateFlag> flag, int bufferSize, short replication, long blockSize,
       Progressable progress) throws IOException ;
   
+  /*.
+   * This create has been added to support the FileContext that processes
+   * the permission
+   * with umask before calling this method.
+   * This a temporary method added to support the transition from FileSystem
+   * to FileContext for user applications.
+   */
+  @Deprecated
+  protected FSDataOutputStream primitiveCreate(Path f,
+     FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
+     short replication, long blockSize, Progressable progress,
+     int bytesPerChecksum) throws IOException {
+    
+    // Default impl  assumes that permissions do not matter and 
+    // nor does the bytesPerChecksum  hence
+    // calling the regular create is good enough.
+    // FSs that implement permissions should override this.
+    
+    return this.create(f, absolutePermission, flag, bufferSize, replication,
+        blockSize, progress);
+  }
+  
+  
+  /*.
+   * This create has been added to support the FileContext that passes
+   * an absolute permission with (ie umask was already applied) 
+   * This a temporary method added to support the transition from FileSystem
+   * to FileContext for user applications.
+   */
+  @Deprecated
+  protected FSDataOutputStream primitiveCreate(final Path f,
+      final EnumSet<CreateFlag> createFlag,
+      CreateOpts... opts) throws IOException {
+    checkPath(f);
+    int bufferSize = -1;
+    short replication = -1;
+    long blockSize = -1;
+    int bytesPerChecksum = -1;
+    FsPermission permission = null;
+    Progressable progress = null;
+    Boolean createParent = null;
+ 
+    for (CreateOpts iOpt : opts) {
+      if (CreateOpts.BlockSize.class.isInstance(iOpt)) {
+        if (blockSize != -1) {
+          throw new IllegalArgumentException("multiple varargs of same kind");
+        }
+        blockSize = ((CreateOpts.BlockSize) iOpt).getValue();
+      } else if (CreateOpts.BufferSize.class.isInstance(iOpt)) {
+        if (bufferSize != -1) {
+          throw new IllegalArgumentException("multiple varargs of same kind");
+        }
+        bufferSize = ((CreateOpts.BufferSize) iOpt).getValue();
+      } else if (CreateOpts.ReplicationFactor.class.isInstance(iOpt)) {
+        if (replication != -1) {
+          throw new IllegalArgumentException("multiple varargs of same kind");
+        }
+        replication = ((CreateOpts.ReplicationFactor) iOpt).getValue();
+      } else if (CreateOpts.BytesPerChecksum.class.isInstance(iOpt)) {
+        if (bytesPerChecksum != -1) {
+          throw new IllegalArgumentException("multiple varargs of same kind");
+        }
+        bytesPerChecksum = ((CreateOpts.BytesPerChecksum) iOpt).getValue();
+      } else if (CreateOpts.Perms.class.isInstance(iOpt)) {
+        if (permission != null) {
+          throw new IllegalArgumentException("multiple varargs of same kind");
+        }
+        permission = ((CreateOpts.Perms) iOpt).getValue();
+      } else if (CreateOpts.Progress.class.isInstance(iOpt)) {
+        if (progress != null) {
+          throw new IllegalArgumentException("multiple varargs of same kind");
+        }
+        progress = ((CreateOpts.Progress) iOpt).getValue();
+      } else if (CreateOpts.CreateParent.class.isInstance(iOpt)) {
+        if (createParent != null) {
+          throw new IllegalArgumentException("multiple varargs of same kind");
+        }
+        createParent = ((CreateOpts.CreateParent) iOpt).getValue();
+      } else {
+        throw new IllegalArgumentException("Unkown CreateOpts of type " +
+            iOpt.getClass().getName());
+      }
+    }
+    if (blockSize % bytesPerChecksum != 0) {
+      throw new IllegalArgumentException(
+          "blockSize should be a multiple of checksumsize");
+    }
+    
+    FsServerDefaults ssDef = getServerDefaults();
+    
+    if (blockSize == -1) {
+      blockSize = ssDef.getBlockSize();
+    }
+    if (bufferSize == -1) {
+      bufferSize = ssDef.getFileBufferSize();
+    }
+    if (replication == -1) {
+      replication = ssDef.getReplication();
+    }
+    if (permission == null) {
+      permission = FsPermission.getDefault();
+    }
+    if (createParent == null) {
+      createParent = false;
+    }
+    
+    // Default impl  assumes that permissions do not matter and 
+    // nor does the bytesPerChecksum  hence
+    // calling the regular create is good enough.
+    // FSs that implement permissions should override this.
+
+    if (!createParent) { // parent must exist.
+      // since this.create makes parent dirs automatically
+      // we must throw exception if parent does not exist.
+      final FileStatus stat = getFileStatus(f.getParent());
+      if (stat == null) {
+        throw new FileNotFoundException("Missing parent:" + f);
+      }
+      if (!stat.isDir()) {
+          throw new ParentNotDirectoryException("parent is not a dir:" + f);
+      }
+      // parent does exist - go ahead with create of file.
+    }
+    return this.create(f, permission, createFlag, bufferSize, replication,
+        blockSize, progress);
+  }
+  
+
+  /**
+   * This version of the mkdirs method assumes that the permission is absolute.
+   * It has been added to support the FileContext that processes the permission
+   * with umask before calling this method.
+   * This a temporary method added to support the transition from FileSystem
+   * to FileContext for user applications.
+   */
+  @Deprecated
+  protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
+    throws IOException {
+    // Default impl is to assume that permissions do not matter and hence
+    // calling the regular mkdirs is good enough.
+    // FSs that implement permissions should override this.
+   return this.mkdirs(f, absolutePermission);
+  }
+
+
+  /**
+   * This version of the mkdirs method assumes that the permission is absolute.
+   * It has been added to support the FileContext that processes the permission
+   * with umask before calling this method.
+   * This a temporary method added to support the transition from FileSystem
+   * to FileContext for user applications.
+   */
+  @Deprecated
+  protected void primitiveMkdir(Path f, FsPermission absolutePermission, 
+                    boolean createParent)
+    throws IOException {
+    
+    if (!createParent) { // parent must exist.
+      // since the this.mkdirs makes parent dirs automatically
+      // we must throw exception if parent does not exist.
+      final FileStatus stat = getFileStatus(f.getParent());
+      if (stat == null) {
+        throw new FileNotFoundException("Missing parent:" + f);
+      }
+      if (!stat.isDir()) {
+          throw new ParentNotDirectoryException("parent is not a dir");
+      }
+      // parent does exist - go ahead with mkdir of leaf
+    }
+    // Default impl is to assume that permissions do not matter and hence
+    // calling the regular mkdirs is good enough.
+    // FSs that implement permissions should override this.
+    if (!this.mkdirs(f, absolutePermission)) {
+      throw new IOException("mkdir of "+ f + " failed");
+    }
+  }
+
 
   /**
    * Creates the given Path as a brand-new zero-length file.  If
@@ -611,9 +832,98 @@
   /**
    * Renames Path src to Path dst.  Can take place on local fs
    * or remote DFS.
+   * @throws IOException on failure
+   * @return true if rename is successful
    */
   public abstract boolean rename(Path src, Path dst) throws IOException;
-    
+
+  /**
+   * Renames Path src to Path dst
+   * <ul>
+   * <li
+   * <li>Fails if src is a file and dst is a directory.
+   * <li>Fails if src is a directory and dst is a file.
+   * <li>Fails if the parent of dst does not exist or is a file.
+   * </ul>
+   * <p>
+   * If OVERWRITE option is not passed as an argument, rename fails
+   * if the dst already exists.
+   * <p>
+   * If OVERWRITE option is passed as an argument, rename overwrites
+   * the dst if it is a file or an empty directory. Rename fails if dst is
+   * a non-empty directory.
+   * <p>
+   * Note that atomicity of rename is dependent on the file system
+   * implementation. Please refer to the file system documentation for
+   * details. This default implementation is non atomic.
+   * <p>
+   * This method is deprecated since it is a temporary method added to 
+   * support the transition from FileSystem to FileContext for user 
+   * applications.
+   * 
+   * @param src path to be renamed
+   * @param dst new path after rename
+   * @throws IOException on failure
+   */
+  @Deprecated
+  protected void rename(final Path src, final Path dst,
+      final Rename... options) throws IOException {
+    // Default implementation
+    final FileStatus srcStatus = getFileStatus(src);
+    if (srcStatus == null) {
+      throw new FileNotFoundException("rename source " + src + " not found.");
+    }
+
+    boolean overwrite = false;
+    if (null != options) {
+      for (Rename option : options) {
+        if (option == Rename.OVERWRITE) {
+          overwrite = true;
+        }
+      }
+    }
+
+    FileStatus dstStatus;
+    try {
+      dstStatus = getFileStatus(dst);
+    } catch (IOException e) {
+      dstStatus = null;
+    }
+    if (dstStatus != null) {
+      if (srcStatus.isDir() != dstStatus.isDir()) {
+        throw new IOException("Source " + src + " Destination " + dst
+            + " both should be either file or directory");
+      }
+      if (!overwrite) {
+        throw new FileAlreadyExistsException("rename destination " + dst
+            + " already exists.");
+      }
+      // Delete the destination that is a file or an empty directory
+      if (dstStatus.isDir()) {
+        FileStatus[] list = listStatus(dst);
+        if (list != null && list.length != 0) {
+          throw new IOException(
+              "rename cannot overwrite non empty destination directory " + dst);
+        }
+      }
+      delete(dst, false);
+    } else {
+      final Path parent = dst.getParent();
+      final FileStatus parentStatus = getFileStatus(parent);
+      if (parentStatus == null) {
+        throw new FileNotFoundException("rename destination parent " + parent
+            + " not found.");
+      }
+      if (!parentStatus.isDir()) {
+        throw new ParentNotDirectoryException("rename destination parent " + parent
+            + " is a file.");
+      }
+    }
+    if (!rename(src, dst)) {
+      throw new IOException("rename from " + src + " to " + dst + " failed.");
+    }
+  }
+  
   /** Delete a file.
    *
    * @param f the path to delete.
@@ -1123,8 +1433,8 @@
    * The default implementation returns "/user/$USER/".
    */
   public Path getHomeDirectory() {
-    return new Path("/user/"+System.getProperty("user.name"))
-      .makeQualified(this);
+    return this.makeQualified(
+        new Path("/user/"+System.getProperty("user.name")));
   }
 
 
@@ -1141,6 +1451,23 @@
    * @return the directory pathname
    */
   public abstract Path getWorkingDirectory();
+  
+  
+  /**
+   * Note: with the new FilesContext class, getWorkingDirectory()
+   * will be removed. 
+   * The working directory is implemented in FilesContext.
+   * 
+   * Some file systems like LocalFileSystem have an initial workingDir
+   * that we use as the starting workingDir. For other file systems
+   * like HDFS there is no built in notion of an inital workingDir.
+   * 
+   * @return if there is built in notion of workingDir then it
+   * is returned; else a null is returned.
+   */
+  protected Path getInitialWorkingDirectory() {
+    return null;
+  }
 
   /**
    * Call {@link #mkdirs(Path, FsPermission)} with default permission.

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileUtil.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FileUtil.java Sat Nov 28 19:53:33 2009
@@ -628,14 +628,18 @@
      * Retrieves the number of links to the specified file.
      */
     public static int getLinkCount(File fileName) throws IOException {
+      if (!fileName.exists()) {
+        throw new FileNotFoundException(fileName + " not found.");
+      }
+
       int len = getLinkCountCommand.length;
       String[] cmd = new String[len + 1];
       for (int i = 0; i < len; i++) {
         cmd[i] = getLinkCountCommand[i];
       }
       cmd[len] = fileName.toString();
-      String inpMsg = "";
-      String errMsg = "";
+      String inpMsg = null;
+      String errMsg = null;
       int exitValue = -1;
       BufferedReader in = null;
       BufferedReader err = null;
@@ -647,14 +651,11 @@
         in = new BufferedReader(new InputStreamReader(
                                     process.getInputStream()));
         inpMsg = in.readLine();
-        if (inpMsg == null)  inpMsg = "";
-        
         err = new BufferedReader(new InputStreamReader(
                                      process.getErrorStream()));
         errMsg = err.readLine();
-        if (errMsg == null)  errMsg = "";
-        if (exitValue != 0) {
-          throw new IOException(inpMsg + errMsg);
+        if (inpMsg == null || exitValue != 0) {
+          throw createIOException(fileName, inpMsg, errMsg, exitValue, null);
         }
         if (getOSType() == OSType.OS_TYPE_SOLARIS) {
           String[] result = inpMsg.split("\\s+");
@@ -663,13 +664,9 @@
           return Integer.parseInt(inpMsg);
         }
       } catch (NumberFormatException e) {
-        throw new IOException(StringUtils.stringifyException(e) + 
-                              inpMsg + errMsg +
-                              " on file:" + fileName);
+        throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
       } catch (InterruptedException e) {
-        throw new IOException(StringUtils.stringifyException(e) + 
-                              inpMsg + errMsg +
-                              " on file:" + fileName);
+        throw createIOException(fileName, inpMsg, errMsg, exitValue, e);
       } finally {
         process.destroy();
         if (in != null) in.close();
@@ -678,6 +675,16 @@
     }
   }
 
+  /** Create an IOException for failing to get link count. */
+  static private IOException createIOException(File f, String message,
+      String error, int exitvalue, Exception cause) {
+    final String s = "Failed to get link count on file " + f
+        + ": message=" + message
+        + "; error=" + error
+        + "; exit value=" + exitvalue;
+    return cause == null? new IOException(s): new IOException(s, cause);
+  }
+
   /**
    * Create a soft link between a src and destination
    * only on a local disk. HDFS does not support this

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FilterFileSystem.java Sat Nov 28 19:53:33 2009
@@ -167,7 +167,11 @@
   public Path getWorkingDirectory() {
     return fs.getWorkingDirectory();
   }
-
+  
+  protected Path getInitialWorkingDirectory() {
+    return fs.getInitialWorkingDirectory();
+  }
+  
   /** {@inheritDoc} */
   @Override
   public FsStatus getStatus(Path p) throws IOException {
@@ -276,4 +280,19 @@
       ) throws IOException {
     fs.setPermission(p, permission);
   }
+
+  @Override
+  protected FSDataOutputStream primitiveCreate(Path f,
+      FsPermission absolutePermission, EnumSet<CreateFlag> flag,
+      int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum)
+      throws IOException {
+    return fs.primitiveCreate(f, absolutePermission, flag,
+        bufferSize, replication, blockSize, progress, bytesPerChecksum);
+  }
+
+  @Override
+  protected boolean primitiveMkdir(Path f, FsPermission abdolutePermission)
+      throws IOException {
+    return fs.primitiveMkdir(f, abdolutePermission);
+  }
 }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShell.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShell.java Sat Nov 28 19:53:33 2009
@@ -367,11 +367,13 @@
     DataOutputBuffer outbuf;
 
     public TextRecordInputStream(FileStatus f) throws IOException {
-      r = new SequenceFile.Reader(fs, f.getPath(), getConf());
-      key = ReflectionUtils.newInstance(r.getKeyClass().asSubclass(WritableComparable.class),
-                                        getConf());
-      val = ReflectionUtils.newInstance(r.getValueClass().asSubclass(Writable.class),
-                                        getConf());
+      final Path fpath = f.getPath();
+      final Configuration lconf = getConf();
+      r = new SequenceFile.Reader(fpath.getFileSystem(lconf), fpath, lconf);
+      key = ReflectionUtils.newInstance(
+          r.getKeyClass().asSubclass(WritableComparable.class), lconf);
+      val = ReflectionUtils.newInstance(
+          r.getValueClass().asSubclass(Writable.class), lconf);
       inbuf = new DataInputBuffer();
       outbuf = new DataOutputBuffer();
     }
@@ -1130,10 +1132,20 @@
     }
     
     if(!skipTrash) {
-      Trash trashTmp = new Trash(srcFs, getConf());
-      if (trashTmp.moveToTrash(src)) {
-        System.out.println("Moved to trash: " + src);
-        return;
+      try {
+	      Trash trashTmp = new Trash(srcFs, getConf());
+        if (trashTmp.moveToTrash(src)) {
+          System.out.println("Moved to trash: " + src);
+          return;
+        }
+      } catch (IOException e) {
+        Exception cause = (Exception) e.getCause();
+        String msg = "";
+        if(cause != null) {
+          msg = cause.getLocalizedMessage();
+        }
+        System.err.println("Problem with Trash." + msg +". Consider using -skipTrash option");        
+        throw e;
       }
     }
     
@@ -1327,7 +1339,7 @@
     String summary = "hadoop fs is the command to execute fs commands. " +
       "The full syntax is: \n\n" +
       "hadoop fs [-fs <local | file system URI>] [-conf <configuration file>]\n\t" +
-      "[-D <property=value>] [-ls <path>] [-lsr <path>] [-df [<path>]] [-du <path>]\n\t" + 
+      "[-D <property=value>] [-ls <path>] [-lsr <path>] [-df [<path>]] [-du [-s] [-h] <path>]\n\t" +
       "[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm [-skipTrash] <src>]\n\t" + 
       "[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
       "[-moveFromLocal <localsrc> ... <dst>] [" + 
@@ -1377,17 +1389,20 @@
       "\t\tIf the filesystem has multiple partitions, and no path to a particular partition\n"+
       "\t\tis specified, then the status of the root partitions will be shown.\n";
 
-    String du = "-du <path>: \tShow the amount of space, in bytes, used by the files that \n" +
-      "\t\tmatch the specified file pattern.  Equivalent to the unix\n" + 
-      "\t\tcommand \"du -sb <path>/*\" in case of a directory, \n" +
-      "\t\tand to \"du -b <path>\" in case of a file.\n" +
+    String du = "-du [-s] [-h] <path>: \tShow the amount of space, in bytes, used by the files that \n" +
+      "\t\tmatch the specified file pattern. The following flags are optional:\n" +
+      "\t\t  -s   Rather than showing the size of each individual file that\n" +
+      "\t\t       matches the pattern, shows the total (summary) size.\n" +
+      "\t\t  -h   Formats the sizes of files in a human-readable fashion\n" +
+      "\t\t       rather than a number of bytes.\n" +
+      "\n" + 
+      "\t\tNote that, even without the -s option, this only shows size summaries\n" +
+      "\t\tone level deep into a directory.\n" +
       "\t\tThe output is in the form \n" + 
-      "\t\t\tname(full path) size (in bytes)\n"; 
+      "\t\t\tsize\tname(full path)\n"; 
 
     String dus = "-dus <path>: \tShow the amount of space, in bytes, used by the files that \n" +
-      "\t\tmatch the specified file pattern.  Equivalent to the unix\n" + 
-      "\t\tcommand \"du -sb\"  The output is in the form \n" + 
-      "\t\t\tname(full path) size (in bytes)\n"; 
+      "\t\tmatch the specified file pattern. This is equivalent to -du -s above.\n";
     
     String mv = "-mv <src> <dst>:   Move files that match the specified file pattern <src>\n" +
       "\t\tto a destination <dst>.  When moving multiple files, the \n" +
@@ -1556,7 +1571,7 @@
       System.out.println(chown);
     } else if ("chgrp".equals(cmd)) {
       System.out.println(chgrp);
-    } else if (Count.matches(cmd)) {
+    } else if (Count.NAME.equals(cmd)) {
       System.out.println(Count.DESCRIPTION);
     } else if ("help".equals(cmd)) {
       System.out.println(help);
@@ -1732,7 +1747,7 @@
       System.err.println("           [-ls <path>]");
       System.err.println("           [-lsr <path>]");
       System.err.println("           [-df [<path>]]");
-      System.err.println("           [-du <path>]");
+      System.err.println("           [-du [-s] [-h] <path>]");
       System.err.println("           [-dus <path>]");
       System.err.println("           [" + Count.USAGE + "]");
       System.err.println("           [-mv <src> <dst>]");

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShellPermissions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShellPermissions.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShellPermissions.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/FsShellPermissions.java Sat Nov 28 19:53:33 2009
@@ -23,6 +23,7 @@
 
 import org.apache.hadoop.fs.FsShell.CmdHandler;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.ChmodParser;
 
 
 /**
@@ -39,168 +40,33 @@
    * also enforce octal mode specifications of either 3 digits without a sticky
    * bit setting or four digits with a sticky bit setting.
    */
-  private static Pattern chmodNormalPattern =
-   Pattern.compile("\\G\\s*([ugoa]*)([+=-]+)([rwxXt]+)([,\\s]*)\\s*");
-  private static Pattern chmodOctalPattern =
-            Pattern.compile("^\\s*[+]?([01]?)([0-7]{3})\\s*$");
 
   static String CHMOD_USAGE =
                             "-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...";
 
+  private static  ChmodParser pp;
+  
   private static class ChmodHandler extends CmdHandler {
 
-    private short userMode;
-    private short groupMode;
-    private short othersMode;
-    private short stickyMode;
-    private char userType = '+';
-    private char groupType = '+';
-    private char othersType = '+';
-    private char stickyBitType = '+';
-
-    private void applyNormalPattern(String modeStr, Matcher matcher)
-                                    throws IOException {
-      // Are there multiple permissions stored in one chmod?
-      boolean commaSeperated = false;
-
-      for(int i=0; i < 1 || matcher.end() < modeStr.length(); i++) {
-        if (i>0 && (!commaSeperated || !matcher.find())) {
-          patternError(modeStr);
-        }
-
-        /* groups : 1 : [ugoa]*
-         *          2 : [+-=]
-         *          3 : [rwxXt]+
-         *          4 : [,\s]*
-         */
-
-        String str = matcher.group(2);
-        char type = str.charAt(str.length() - 1);
-
-        boolean user, group, others, stickyBit;
-        user = group = others = stickyBit = false;
-
-        for(char c : matcher.group(1).toCharArray()) {
-          switch (c) {
-          case 'u' : user = true; break;
-          case 'g' : group = true; break;
-          case 'o' : others = true; break;
-          case 'a' : break;
-          default  : throw new RuntimeException("Unexpected");          
-          }
-        }
-
-        if (!(user || group || others)) { // same as specifying 'a'
-          user = group = others = true;
-        }
-
-        short mode = 0;
-
-        for(char c : matcher.group(3).toCharArray()) {
-          switch (c) {
-          case 'r' : mode |= 4; break;
-          case 'w' : mode |= 2; break;
-          case 'x' : mode |= 1; break;
-          case 'X' : mode |= 8; break;
-          case 't' : stickyBit = true; break;
-          default  : throw new RuntimeException("Unexpected");
-          }
-        }
-
-        if ( user ) {
-          userMode = mode;
-          userType = type;
-        }
-
-        if ( group ) {
-          groupMode = mode;
-          groupType = type;
-        }
-
-        if ( others ) {
-          othersMode = mode;
-          othersType = type;
-          
-          stickyMode = (short) (stickyBit ? 1 : 0);
-          stickyBitType = type;
-        }
-
-        commaSeperated = matcher.group(4).contains(",");
-      }
-    }
-
-    private void applyOctalPattern(String modeStr, Matcher matcher) {
-      userType = groupType = othersType = '=';
-
-      // Check if sticky bit is specified
-      String sb = matcher.group(1);
-      if(!sb.isEmpty()) {
-        stickyMode = Short.valueOf(sb.substring(0, 1));
-        stickyBitType = '=';
-      }
-
-      String str = matcher.group(2);
-      userMode = Short.valueOf(str.substring(0, 1));
-      groupMode = Short.valueOf(str.substring(1, 2));
-      othersMode = Short.valueOf(str.substring(2, 3));      
-    }
-
-    private void patternError(String mode) throws IOException {
-      throw new IOException("chmod : mode '" + mode + 
-                            "' does not match the expected pattern.");      
-    }
-
     ChmodHandler(FileSystem fs, String modeStr) throws IOException {
       super("chmod", fs);
-      Matcher matcher = null;
-
-      if ((matcher = chmodNormalPattern.matcher(modeStr)).find()) {
-        applyNormalPattern(modeStr, matcher);
-      } else if ((matcher = chmodOctalPattern.matcher(modeStr)).matches()) {
-        applyOctalPattern(modeStr, matcher);
-      } else {
-        patternError(modeStr);
+      try {
+        pp = new ChmodParser(modeStr);
+      } catch(IllegalArgumentException iea) {
+        patternError(iea.getMessage());
       }
     }
 
-    private int applyChmod(char type, int mode, int existing, boolean exeOk) {
-      boolean capX = false;
-
-      if ((mode&8) != 0) { // convert X to x;
-        capX = true;
-        mode &= ~8;
-        mode |= 1;
-      }
-
-      switch (type) {
-      case '+' : mode = mode | existing; break;
-      case '-' : mode = (~mode) & existing; break;
-      case '=' : break;
-      default  : throw new RuntimeException("Unexpected");      
-      }
-
-      // if X is specified add 'x' only if exeOk or x was already set.
-      if (capX && !exeOk && (mode&1) != 0 && (existing&1) == 0) {
-        mode &= ~1; // remove x
-      }
-
-      return mode;
+    private void patternError(String mode) throws IOException {
+     throw new IOException("chmod : mode '" + mode + 
+         "' does not match the expected pattern.");      
     }
-
+    
     @Override
     public void run(FileStatus file, FileSystem srcFs) throws IOException {
-      FsPermission perms = file.getPermission();
-      int existing = perms.toShort();
-      boolean exeOk = file.isDir() || (existing & 0111) != 0;
-      int newperms = ( applyChmod(stickyBitType, stickyMode,
-                             (existing>>>9), false) << 9 |
-                       applyChmod(userType, userMode,
-                             (existing>>>6)&7, exeOk) << 6 |
-                       applyChmod(groupType, groupMode,
-                             (existing>>>3)&7, exeOk) << 3 |
-                       applyChmod(othersType, othersMode, existing&7, exeOk));
+      int newperms = pp.applyNewPermission(file);
 
-      if (existing != newperms) {
+      if (file.getPermission().toShort() != newperms) {
         try {
           srcFs.setPermission(file.getPath(), 
                                 new FsPermission((short)newperms));

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/HarFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/HarFileSystem.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/HarFileSystem.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/HarFileSystem.java Sat Nov 28 19:53:33 2009
@@ -302,19 +302,8 @@
     }
 
     URI tmpURI = fsPath.toUri();
-    fsPath = new Path(tmpURI.getPath());
     //change this to Har uri 
-    URI tmp = null;
-    try {
-      tmp = new URI(uri.getScheme(), harAuth, fsPath.toString(),
-                    tmpURI.getQuery(), tmpURI.getFragment());
-    } catch(URISyntaxException ue) {
-      LOG.error("Error in URI ", ue);
-    }
-    if (tmp != null) {
-      return new Path(tmp.toString());
-    }
-    return null;
+    return new Path(uri.getScheme(), harAuth, tmpURI.getPath());
   }
   
   /**
@@ -426,12 +415,13 @@
       // do nothing just a read.
     }
     FSDataInputStream aIn = fs.open(archiveIndex);
-    LineReader aLin = new LineReader(aIn, getConf());
+    LineReader aLin;
     String retStr = null;
     // now start reading the real index file
-     read = 0;
     for (Store s: stores) {
+      read = 0;
       aIn.seek(s.begin);
+      aLin = new LineReader(aIn, getConf());
       while (read + s.begin < s.end) {
         int tmp = aLin.readLine(line);
         read += tmp;

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Path.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Path.java Sat Nov 28 19:53:33 2009
@@ -63,13 +63,13 @@
     if (!(parentPath.equals("/") || parentPath.equals("")))
       try {
         parentUri = new URI(parentUri.getScheme(), parentUri.getAuthority(),
-                            parentUri.getPath()+"/", null, null);
+                      parentUri.getPath()+"/", null, parentUri.getFragment());
       } catch (URISyntaxException e) {
         throw new IllegalArgumentException(e);
       }
     URI resolved = parentUri.resolve(child.uri);
     initialize(resolved.getScheme(), resolved.getAuthority(),
-               normalizePath(resolved.getPath()));
+               normalizePath(resolved.getPath()), resolved.getFragment());
   }
 
   private void checkPathArg( String path ) {
@@ -123,18 +123,26 @@
     // uri path is the rest of the string -- query & fragment not supported
     String path = pathString.substring(start, pathString.length());
 
-    initialize(scheme, authority, path);
+    initialize(scheme, authority, path, null);
   }
 
+  /**
+   * Construct a path from a URI
+   */
+  public Path(URI aUri) {
+    uri = aUri;
+  }
+  
   /** Construct a Path from components. */
   public Path(String scheme, String authority, String path) {
     checkPathArg( path );
-    initialize(scheme, authority, path);
+    initialize(scheme, authority, path, null);
   }
 
-  private void initialize(String scheme, String authority, String path) {
+  private void initialize(String scheme, String authority, String path,
+      String fragment) {
     try {
-      this.uri = new URI(scheme, authority, normalizePath(path), null, null)
+      this.uri = new URI(scheme, authority, normalizePath(path), null, fragment)
         .normalize();
     } catch (URISyntaxException e) {
       throw new IllegalArgumentException(e);
@@ -175,10 +183,23 @@
     return FileSystem.get(this.toUri(), conf);
   }
 
-  /** True if the directory of this path is absolute. */
-  public boolean isAbsolute() {
+  /**
+   *  True if the path component (i.e. directory) of this URI is absolute.
+   */
+  public boolean isUriPathAbsolute() {
     int start = hasWindowsDrive(uri.getPath(), true) ? 3 : 0;
     return uri.getPath().startsWith(SEPARATOR, start);
+   }
+  
+  /** True if the directory of this path is absolute. */
+  /**
+   * There is some ambiguity here. An absolute path is a slash
+   * relative name without a scheme or an authority.
+   * So either this method was incorrectly named or its
+   * implementation is incorrect.
+   */
+  public boolean isAbsolute() {
+     return isUriPathAbsolute();
   }
 
   /** Returns the final component of this path.*/
@@ -233,6 +254,10 @@
         path = path.substring(1);                 // remove slash before drive
       buffer.append(path);
     }
+    if (uri.getFragment() != null) {
+      buffer.append("#");
+      buffer.append(uri.getFragment());
+    }
     return buffer.toString();
   }
 
@@ -265,34 +290,54 @@
     return depth;
   }
 
-  /** Returns a qualified path object. */
+  
+  /**
+   *  Returns a qualified path object.
+   *  
+   *  Deprecated - use {@link #makeQualified(URI, Path)}
+   */
+ 
+  @Deprecated
   public Path makeQualified(FileSystem fs) {
+    return makeQualified(fs.getUri(), fs.getWorkingDirectory());
+  }
+  
+  
+  /** Returns a qualified path object. */
+  public Path makeQualified(URI defaultUri, Path workingDir ) {
     Path path = this;
     if (!isAbsolute()) {
-      path = new Path(fs.getWorkingDirectory(), this);
+      path = new Path(workingDir, this);
     }
 
     URI pathUri = path.toUri();
-    URI fsUri = fs.getUri();
       
     String scheme = pathUri.getScheme();
     String authority = pathUri.getAuthority();
+    String fragment = pathUri.getFragment();
 
     if (scheme != null &&
-        (authority != null || fsUri.getAuthority() == null))
+        (authority != null || defaultUri.getAuthority() == null))
       return path;
 
     if (scheme == null) {
-      scheme = fsUri.getScheme();
+      scheme = defaultUri.getScheme();
     }
 
     if (authority == null) {
-      authority = fsUri.getAuthority();
+      authority = defaultUri.getAuthority();
       if (authority == null) {
         authority = "";
       }
     }
-
-    return new Path(scheme+":"+"//"+authority + pathUri.getPath());
+    
+    URI newUri = null;
+    try {
+      newUri = new URI(scheme, authority , 
+        normalizePath(pathUri.getPath()), null, fragment);
+    } catch (URISyntaxException e) {
+      throw new IllegalArgumentException(e);
+    }
+    return new Path(newUri);
   }
 }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Sat Nov 28 19:53:33 2009
@@ -46,7 +46,7 @@
   private Path workingDir;
   
   public RawLocalFileSystem() {
-    workingDir = new Path(System.getProperty("user.dir")).makeQualified(this);
+    workingDir = getInitialWorkingDirectory();
   }
   
   /** Convert a path to a File. */
@@ -96,10 +96,10 @@
   }
 
   /*******************************************************
-   * For open()'s FSInputStream
+   * For open()'s FSInputStream.
    *******************************************************/
   class LocalFSFileInputStream extends FSInputStream {
-    FileInputStream fis;
+    private FileInputStream fis;
     private long position;
 
     public LocalFSFileInputStream(Path f) throws IOException {
@@ -180,8 +180,8 @@
   /*********************************************************
    * For create()'s FSOutputStream.
    *********************************************************/
-  class LocalFSFileOutputStream extends OutputStream implements Syncable {
-    FileOutputStream fos;
+  class LocalFSFileOutputStream extends OutputStream {
+    private FileOutputStream fos;
     
     private LocalFSFileOutputStream(Path f, boolean append) throws IOException {
       this.fos = new FileOutputStream(pathToFile(f), append);
@@ -207,13 +207,8 @@
         throw new FSError(e);                // assume native fs error
       }
     }
-
-    /** {@inheritDoc} */
-    public void sync() throws IOException {
-      fos.getFD().sync();      
-    }
   }
-  
+
   /** {@inheritDoc} */
   public FSDataOutputStream append(Path f, int bufferSize,
       Progressable progress) throws IOException {
@@ -229,7 +224,7 @@
 
   /** {@inheritDoc} */
   public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize,
-                                   short replication, long blockSize, Progressable progress)
+    short replication, long blockSize, Progressable progress)
     throws IOException {
     if (exists(f) && !overwrite) {
       throw new IOException("File already exists:"+f);
@@ -245,23 +240,38 @@
   /** {@inheritDoc} */
   @Override
   public FSDataOutputStream create(Path f, FsPermission permission,
-      EnumSet<CreateFlag> flag, int bufferSize, short replication, long blockSize,
-      Progressable progress) throws IOException {
+    EnumSet<CreateFlag> flag, int bufferSize, short replication, long blockSize,
+    Progressable progress) throws IOException {
+    return primitiveCreate(f,
+        permission.applyUMask(FsPermission.getUMask(getConf())), flag,
+        bufferSize,  replication,  blockSize,  progress,  -1);
     
-      if(flag.contains(CreateFlag.APPEND)){
-        if (!exists(f)){
-          if(flag.contains(CreateFlag.CREATE))
-            return create(f, false, bufferSize, replication, blockSize, progress);
+    
+     
+  }
+  
+
+  @Override
+  protected FSDataOutputStream primitiveCreate(Path f,
+      FsPermission absolutePermission, EnumSet<CreateFlag> flag,
+      int bufferSize, short replication, long blockSize, Progressable progress,
+      int bytesPerChecksum) throws IOException {
+    
+    if(flag.contains(CreateFlag.APPEND)){
+      if (!exists(f)){
+        if(flag.contains(CreateFlag.CREATE)) {
+          return create(f, false, bufferSize, replication, blockSize, null);
         }
-        return append(f, bufferSize, progress);
+      }
+      return append(f, bufferSize, null);
     }
-   
-    FSDataOutputStream out = create(f,
-        flag.contains(CreateFlag.OVERWRITE), bufferSize, replication, blockSize, progress);
-    setPermission(f, permission);
+ 
+    FSDataOutputStream out = create(f, flag.contains(CreateFlag.OVERWRITE),
+                                 bufferSize, replication, blockSize, progress);
+    setPermission(f, absolutePermission);
     return out;
   }
-  
+
   public boolean rename(Path src, Path dst) throws IOException {
     if (pathToFile(src).renameTo(pathToFile(dst))) {
       return true;
@@ -289,7 +299,7 @@
     }
     if (localf.isFile()) {
       return new FileStatus[] {
-          new RawLocalFileStatus(localf, getDefaultBlockSize(), this) };
+        new RawLocalFileStatus(localf, getDefaultBlockSize(), this) };
     }
 
     String[] names = localf.list();
@@ -308,8 +318,18 @@
    * treat existence as an error.
    */
   public boolean mkdirs(Path f) throws IOException {
+    if(f == null) {
+      throw new IllegalArgumentException("mkdirs path arg is null");
+    }
     Path parent = f.getParent();
     File p2f = pathToFile(f);
+    if(parent != null) {
+      File parent2f = pathToFile(parent);
+      if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
+        throw new FileAlreadyExistsException("Parent path is not a directory: " 
+            + parent);
+      }
+    }
     return (parent == null || mkdirs(parent)) &&
       (p2f.mkdir() || p2f.isDirectory());
   }
@@ -318,13 +338,25 @@
   @Override
   public boolean mkdirs(Path f, FsPermission permission) throws IOException {
     boolean b = mkdirs(f);
-    setPermission(f, permission);
+    if(b) {
+      setPermission(f, permission);
+    }
+    return b;
+  }
+  
+
+  @Override
+  protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
+    throws IOException {
+    boolean b = mkdirs(f);
+    setPermission(f, absolutePermission);
     return b;
   }
   
+  
   @Override
   public Path getHomeDirectory() {
-    return new Path(System.getProperty("user.home")).makeQualified(this);
+    return this.makeQualified(new Path(System.getProperty("user.home")));
   }
 
   /**
@@ -339,6 +371,11 @@
   public Path getWorkingDirectory() {
     return workingDir;
   }
+  
+  @Override
+  protected Path getInitialWorkingDirectory() {
+    return this.makeQualified(new Path(System.getProperty("user.dir")));
+  }
 
   /** {@inheritDoc} */
   @Override
@@ -380,7 +417,7 @@
     if (path.exists()) {
       return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this);
     } else {
-      throw new FileNotFoundException( "File " + f + " does not exist.");
+      throw new FileNotFoundException("File " + f + " does not exist.");
     }
   }
 
@@ -395,7 +432,7 @@
     
     RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) {
       super(f.length(), f.isDirectory(), 1, defaultBlockSize,
-            f.lastModified(), new Path(f.getPath()).makeQualified(fs));
+            f.lastModified(), fs.makeQualified(new Path(f.getPath())));
     }
     
     @Override
@@ -471,8 +508,8 @@
    * Use the command chown to set owner.
    */
   @Override
-  public void setOwner(Path p, String username, String groupname
-      ) throws IOException {
+  public void setOwner(Path p, String username, String groupname)
+    throws IOException {
     if (username == null && groupname == null) {
       throw new IOException("username == null && groupname == null");
     }
@@ -490,8 +527,8 @@
    * Use the command chmod to set permission.
    */
   @Override
-  public void setPermission(Path p, FsPermission permission
-      ) throws IOException {
+  public void setPermission(Path p, FsPermission permission)
+    throws IOException {
     execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND,
         String.format("%05o", permission.toShort()));
   }
@@ -503,4 +540,5 @@
     String output = Shell.execCommand(args);
     return output;
   }
+
 }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Syncable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Syncable.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Syncable.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Syncable.java Sat Nov 28 19:53:33 2009
@@ -20,11 +20,23 @@
 
 import java.io.IOException;
 
-/** This interface declare the sync() operation. */
+/** This interface for flush/sync operation. */
 public interface Syncable {
   /**
-   * Synchronize all buffer with the underlying devices.
-   * @throws IOException
+   * @deprecated As of HADOOP 0.21.0, replaced by hflush
+   * @see #hflush()
    */
-  public void sync() throws IOException;
+  @Deprecated  public void sync() throws IOException;
+  
+  /** Flush out the data in client's user buffer. After the return of
+   * this call, new readers will see the data.
+   * @throws IOException if any error occurs
+   */
+  public void hflush() throws IOException;
+  
+  /** Similar to posix fsync, flush out the data in client's user buffer 
+   * all the way to the disk device (but the disk may have it in its cache).
+   * @throws IOException if error occurs
+   */
+  public void hsync() throws IOException;
 }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Trash.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Trash.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Trash.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/Trash.java Sat Nov 28 19:53:33 2009
@@ -123,12 +123,13 @@
     for (int i = 0; i < 2; i++) {
       try {
         if (!fs.mkdirs(baseTrashPath, PERMISSION)) {      // create current
-          LOG.warn("Can't create trash directory: "+baseTrashPath);
+          LOG.warn("Can't create(mkdir) trash directory: "+baseTrashPath);
           return false;
         }
       } catch (IOException e) {
         LOG.warn("Can't create trash directory: "+baseTrashPath);
-        return false;
+        cause = e;
+        break;
       }
       try {
         //

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/permission/FsPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/permission/FsPermission.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/permission/FsPermission.java Sat Nov 28 19:53:33 2009
@@ -17,17 +17,24 @@
  */
 package org.apache.hadoop.fs.permission;
 
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.*;
-
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableFactory;
+
 /**
  * A class for file/directory permissions.
  */
 public class FsPermission implements Writable {
+  private static final Log LOG = LogFactory.getLog(FsPermission.class);
+  
   static final WritableFactory FACTORY = new WritableFactory() {
     public Writable newInstance() { return new FsPermission(); }
   };
@@ -175,16 +182,29 @@
         otheraction.and(umask.otheraction.not()));
   }
 
-  /** umask property label */
-  public static final String UMASK_LABEL = "dfs.umask";
-  public static final int DEFAULT_UMASK = 0022;
+  /** umask property label Deprecated key may be removed in version .23 */
+  public static final String DEPRECATED_UMASK_LABEL = "dfs.umask"; 
+  public static final String UMASK_LABEL = 
+                  CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY;
+  public static final int DEFAULT_UMASK = 
+                  CommonConfigurationKeys.FS_PERMISSIONS_UMASK_DEFAULT;
 
   /** Get the user file creation mask (umask) */
   public static FsPermission getUMask(Configuration conf) {
     int umask = DEFAULT_UMASK;
-    if (conf != null) {
-      umask = conf.getInt(UMASK_LABEL, DEFAULT_UMASK);
+    
+    // Attempt to pull value from configuration, trying new key first and then
+    // deprecated key, along with a warning, if not present
+    if(conf != null) {
+      String confUmask = conf.get(UMASK_LABEL);
+      if(confUmask != null) { // UMASK_LABEL is set
+        if(conf.deprecatedKeyWasSet(DEPRECATED_UMASK_LABEL)) 
+          umask = Integer.parseInt(confUmask); // Evaluate as decimal value
+        else
+          umask = new UmaskParser(confUmask).getUMask();
+      } 
     }
+    
     return new FsPermission((short)umask);
   }
 

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java Sat Nov 28 19:53:33 2009
@@ -98,7 +98,10 @@
     }
     bucket = new S3Bucket(uri.getHost());
 
-    this.bufferSize = conf.getInt("io.file.buffer.size", 4096);
+    this.bufferSize = conf.getInt(
+                       S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_KEY,
+                       S3FileSystemConfigKeys.S3_STREAM_BUFFER_SIZE_DEFAULT
+		      );
   }
 
   public String getVersion() throws IOException {

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Sat Nov 28 19:53:33 2009
@@ -455,6 +455,7 @@
     } while (priorLastKey != null);
     
     if (status.isEmpty() &&
+        key.length() > 0 &&
         store.retrieveMetadata(key + FOLDER_SUFFIX) == null) {
       throw new FileNotFoundException("File " + f + " does not exist.");
     }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/FilterInitializer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/FilterInitializer.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/FilterInitializer.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/FilterInitializer.java Sat Nov 28 19:53:33 2009
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.http;
 
+import org.apache.hadoop.conf.Configuration;
+
 /**
  * Initialize a javax.servlet.Filter. 
  */
@@ -24,6 +26,7 @@
   /**
    * Initialize a Filter to a FilterContainer.
    * @param container The filter container
+   * @param conf Configuration for run-time parameters
    */
-  abstract void initFilter(FilterContainer container);
+  public abstract void initFilter(FilterContainer container, Configuration conf);
 }
\ No newline at end of file

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/HttpServer.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/http/HttpServer.java Sat Nov 28 19:53:33 2009
@@ -23,14 +23,20 @@
 import java.net.InetSocketAddress;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.nio.channels.ServerSocketChannel;
 
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
 import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.commons.logging.Log;
@@ -117,10 +123,11 @@
 
     addDefaultApps(contexts, appDir);
 
+    addGlobalFilter("safety", QuotingInputFilter.class.getName(), null);
     final FilterInitializer[] initializers = getFilterInitializers(conf); 
     if (initializers != null) {
       for(FilterInitializer c : initializers) {
-        c.initFilter(this);
+        c.initFilter(this, conf);
       }
     }
     addDefaultServlets();
@@ -535,10 +542,103 @@
     public void doGet(HttpServletRequest request, HttpServletResponse response)
       throws ServletException, IOException {
       
-      PrintWriter out = new PrintWriter(response.getOutputStream());
+      PrintWriter out = new PrintWriter
+                    (HtmlQuoting.quoteOutputStream(response.getOutputStream()));
       ReflectionUtils.printThreadInfo(out, "");
       out.close();
       ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);      
     }
   }
+  
+  /**
+   * A Servlet input filter that quotes all HTML active characters in the
+   * parameter names and values. The goal is to quote the characters to make
+   * all of the servlets resistant to cross-site scripting attacks.
+   */
+  public static class QuotingInputFilter implements Filter {
+
+    public static class RequestQuoter extends HttpServletRequestWrapper {
+      private final HttpServletRequest rawRequest;
+      public RequestQuoter(HttpServletRequest rawRequest) {
+        super(rawRequest);
+        this.rawRequest = rawRequest;
+      }
+      
+      /**
+       * Return the set of parameter names, quoting each name.
+       */
+      @SuppressWarnings("unchecked")
+      @Override
+      public Enumeration<String> getParameterNames() {
+        return new Enumeration<String>() {
+          private Enumeration<String> rawIterator = 
+            rawRequest.getParameterNames();
+          @Override
+          public boolean hasMoreElements() {
+            return rawIterator.hasMoreElements();
+          }
+
+          @Override
+          public String nextElement() {
+            return HtmlQuoting.quoteHtmlChars(rawIterator.nextElement());
+          }
+        };
+      }
+      
+      /**
+       * Unquote the name and quote the value.
+       */
+      @Override
+      public String getParameter(String name) {
+        return HtmlQuoting.quoteHtmlChars(rawRequest.getParameter
+                                     (HtmlQuoting.unquoteHtmlChars(name)));
+      }
+      
+      @Override
+      public String[] getParameterValues(String name) {
+        String unquoteName = HtmlQuoting.unquoteHtmlChars(name);
+        String[] unquoteValue = rawRequest.getParameterValues(unquoteName);
+        String[] result = new String[unquoteValue.length];
+        for(int i=0; i < result.length; ++i) {
+          result[i] = HtmlQuoting.quoteHtmlChars(unquoteValue[i]);
+        }
+        return result;
+      }
+
+      @SuppressWarnings("unchecked")
+      @Override
+      public Map<String, String[]> getParameterMap() {
+        Map<String, String[]> result = new HashMap<String,String[]>();
+        Map<String, String[]> raw = rawRequest.getParameterMap();
+        for (Map.Entry<String,String[]> item: raw.entrySet()) {
+          String[] rawValue = item.getValue();
+          String[] cookedValue = new String[rawValue.length];
+          for(int i=0; i< rawValue.length; ++i) {
+            cookedValue[i] = HtmlQuoting.quoteHtmlChars(rawValue[i]);
+          }
+          result.put(HtmlQuoting.quoteHtmlChars(item.getKey()), cookedValue);
+        }
+        return result;
+      }
+    }
+
+    @Override
+    public void init(FilterConfig config) throws ServletException {
+    }
+
+    @Override
+    public void destroy() {
+    }
+
+    @Override
+    public void doFilter(ServletRequest request, 
+                         ServletResponse response,
+                         FilterChain chain
+                         ) throws IOException, ServletException {
+      HttpServletRequestWrapper quoted = 
+        new RequestQuoter((HttpServletRequest) request);
+      chain.doFilter(quoted, response);
+    }
+
+  }
 }

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/DefaultStringifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/DefaultStringifier.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/DefaultStringifier.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/DefaultStringifier.java Sat Nov 28 19:53:33 2009
@@ -21,20 +21,21 @@
 import java.io.IOException;
 import java.nio.charset.UnsupportedCharsetException;
 import java.util.ArrayList;
+import java.util.Map;
 
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.serializer.Deserializer;
-import org.apache.hadoop.io.serializer.Serialization;
+import org.apache.hadoop.io.serializer.DeserializerBase;
+import org.apache.hadoop.io.serializer.SerializationBase;
 import org.apache.hadoop.io.serializer.SerializationFactory;
-import org.apache.hadoop.io.serializer.Serializer;
+import org.apache.hadoop.io.serializer.SerializerBase;
 import org.apache.hadoop.util.GenericsUtil;
 
 /**
  * DefaultStringifier is the default implementation of the {@link Stringifier}
  * interface which stringifies the objects using base64 encoding of the
- * serialized version of the objects. The {@link Serializer} and
- * {@link Deserializer} are obtained from the {@link SerializationFactory}.
+ * serialized version of the objects. The {@link SerializerBase} and
+ * {@link DeserializerBase} are obtained from the {@link SerializationFactory}.
  * <br>
  * DefaultStringifier offers convenience methods to store/load objects to/from
  * the configuration.
@@ -45,9 +46,9 @@
 
   private static final String SEPARATOR = ",";
 
-  private Serializer<T> serializer;
+  private SerializerBase<T> serializer;
 
-  private Deserializer<T> deserializer;
+  private DeserializerBase<T> deserializer;
 
   private DataInputBuffer inBuf;
 
@@ -56,8 +57,9 @@
   public DefaultStringifier(Configuration conf, Class<T> c) {
 
     SerializationFactory factory = new SerializationFactory(conf);
-    this.serializer = factory.getSerializer(c);
-    this.deserializer = factory.getDeserializer(c);
+    Map<String, String> metadata = SerializationBase.getMetadataFromClass(c);
+    this.serializer = factory.getSerializer(metadata);
+    this.deserializer = factory.getDeserializer(metadata);
     this.inBuf = new DataInputBuffer();
     this.outBuf = new DataOutputBuffer();
     try {
@@ -102,7 +104,7 @@
    * @param item the object to be stored
    * @param keyName the name of the key to use
    * @throws IOException : forwards Exceptions from the underlying 
-   * {@link Serialization} classes. 
+   * {@link SerializationBase} classes. 
    */
   public static <K> void store(Configuration conf, K item, String keyName)
   throws IOException {
@@ -122,7 +124,7 @@
    * @param itemClass the class of the item
    * @return restored object
    * @throws IOException : forwards Exceptions from the underlying 
-   * {@link Serialization} classes.
+   * {@link SerializationBase} classes.
    */
   public static <K> K load(Configuration conf, String keyName,
       Class<K> itemClass) throws IOException {
@@ -145,7 +147,7 @@
    * @param keyName the name of the key to use
    * @throws IndexOutOfBoundsException if the items array is empty
    * @throws IOException : forwards Exceptions from the underlying 
-   * {@link Serialization} classes.         
+   * {@link SerializationBase} classes.         
    */
   public static <K> void storeArray(Configuration conf, K[] items,
       String keyName) throws IOException {
@@ -173,7 +175,7 @@
    * @param itemClass the class of the item
    * @return restored object
    * @throws IOException : forwards Exceptions from the underlying 
-   * {@link Serialization} classes.
+   * {@link SerializationBase} classes.
    */
   public static <K> K[] loadArray(Configuration conf, String keyName,
       Class<K> itemClass) throws IOException {

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/SequenceFile.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/SequenceFile.java Sat Nov 28 19:53:33 2009
@@ -33,9 +33,10 @@
 import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
-import org.apache.hadoop.io.serializer.Deserializer;
+import org.apache.hadoop.io.serializer.DeserializerBase;
+import org.apache.hadoop.io.serializer.SerializationBase;
+import org.apache.hadoop.io.serializer.SerializerBase;
 import org.apache.hadoop.io.serializer.SerializationFactory;
-import org.apache.hadoop.io.serializer.Serializer;
 import org.apache.hadoop.conf.*;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Progress;
@@ -705,6 +706,14 @@
       return new TreeMap<Text, Text>(this.theMetadata);
     }
     
+    public Map<String, String> getMetadataAsStringMap() {
+      Map<String, String> map = new HashMap<String, String>();
+      for (Map.Entry<Text, Text> entry : theMetadata.entrySet()) {
+        map.put(entry.getKey().toString(), entry.getValue().toString());
+      }
+      return map;
+    }
+    
     public void write(DataOutput out) throws IOException {
       out.writeInt(this.theMetadata.size());
       Iterator<Map.Entry<Text, Text>> iter =
@@ -801,9 +810,9 @@
     Metadata metadata = null;
     Compressor compressor = null;
     
-    protected Serializer keySerializer;
-    protected Serializer uncompressedValSerializer;
-    protected Serializer compressedValSerializer;
+    protected SerializerBase keySerializer;
+    protected SerializerBase uncompressedValSerializer;
+    protected SerializerBase compressedValSerializer;
     
     // Insert a globally unique 16-byte value every few entries, so that one
     // can seek into the middle of a file and then synchronize with record
@@ -914,9 +923,10 @@
       this.codec = codec;
       this.metadata = metadata;
       SerializationFactory serializationFactory = new SerializationFactory(conf);
-      this.keySerializer = serializationFactory.getSerializer(keyClass);
+      this.keySerializer = getSerializer(serializationFactory, keyClass, metadata);
       this.keySerializer.open(buffer);
-      this.uncompressedValSerializer = serializationFactory.getSerializer(valClass);
+      this.uncompressedValSerializer = getSerializer(serializationFactory,
+        valClass, metadata);
       this.uncompressedValSerializer.open(buffer);
       if (this.codec != null) {
         ReflectionUtils.setConf(this.codec, this.conf);
@@ -924,11 +934,20 @@
         this.deflateFilter = this.codec.createOutputStream(buffer, compressor);
         this.deflateOut = 
           new DataOutputStream(new BufferedOutputStream(deflateFilter));
-        this.compressedValSerializer = serializationFactory.getSerializer(valClass);
+        this.compressedValSerializer = getSerializer(serializationFactory,
+          valClass, metadata);
         this.compressedValSerializer.open(deflateOut);
       }
     }
     
+    @SuppressWarnings("unchecked")
+    private SerializerBase getSerializer(SerializationFactory sf, Class c,
+	Metadata metadata) {
+      Map<String, String> stringMetadata = metadata.getMetadataAsStringMap();
+      stringMetadata.put(SerializationBase.CLASS_KEY, c.getName());
+      return sf.getSerializer(stringMetadata);
+    }
+    
     /** Returns the class of keys in this file. */
     public Class getKeyClass() { return keyClass; }
 
@@ -1378,6 +1397,7 @@
     private byte[] syncCheck = new byte[SYNC_HASH_SIZE];
     private boolean syncSeen;
 
+    private long headerEnd;
     private long end;
     private int keyLength;
     private int recordLength;
@@ -1412,8 +1432,8 @@
     private DataInputStream valIn = null;
     private Decompressor valDecompressor = null;
     
-    private Deserializer keyDeserializer;
-    private Deserializer valDeserializer;
+    private DeserializerBase keyDeserializer;
+    private DeserializerBase valDeserializer;
 
     /** Open the named file. */
     public Reader(FileSystem fs, Path file, Configuration conf)
@@ -1527,6 +1547,7 @@
       
       if (version > 1) {                          // if version > 1
         in.readFully(sync);                       // read sync bytes
+        headerEnd = in.getPos();                  // record end of header
       }
       
       // Initialize... *not* if this we are constructing a temporary Reader
@@ -1563,21 +1584,24 @@
         SerializationFactory serializationFactory =
           new SerializationFactory(conf);
         this.keyDeserializer =
-          getDeserializer(serializationFactory, getKeyClass());
+          getDeserializer(serializationFactory, getKeyClass(), metadata);
         if (!blockCompressed) {
           this.keyDeserializer.open(valBuffer);
         } else {
           this.keyDeserializer.open(keyIn);
         }
         this.valDeserializer =
-          getDeserializer(serializationFactory, getValueClass());
+          getDeserializer(serializationFactory, getValueClass(), metadata);
         this.valDeserializer.open(valIn);
       }
     }
     
     @SuppressWarnings("unchecked")
-    private Deserializer getDeserializer(SerializationFactory sf, Class c) {
-      return sf.getDeserializer(c);
+    private DeserializerBase getDeserializer(SerializationFactory sf, Class c,
+	Metadata metadata) {
+      Map<String, String> stringMetadata = metadata.getMetadataAsStringMap();
+      stringMetadata.put(SerializationBase.CLASS_KEY, c.getName());
+      return sf.getDeserializer(stringMetadata);
     }
     
     /** Close the file. */
@@ -2188,6 +2212,14 @@
         return;
       }
 
+      if (position < headerEnd) {
+        // seek directly to first record
+        in.seek(headerEnd);
+        // note the sync marker "seen" in the header
+        syncSeen = true;
+        return;
+      }
+
       try {
         seek(position+4);                         // skip escape
         in.readFully(syncCheck);

Modified: hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/WritableUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/WritableUtils.java?rev=885142&r1=885141&r2=885142&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/WritableUtils.java (original)
+++ hadoop/common/branches/HADOOP-6194/src/java/org/apache/hadoop/io/WritableUtils.java Sat Nov 28 19:53:33 2009
@@ -415,4 +415,29 @@
     }
     return out.getData();
   }
+
+  /**
+   * Read a string, but check it for sanity. The format consists of a vint
+   * followed by the given number of bytes.
+   * @param in the stream to read from
+   * @param maxLength the largest acceptable length of the encoded string
+   * @return the bytes as a string
+   * @throws IOException if reading from the DataInput fails
+   * @throws IllegalArgumentException if the encoded byte size for string 
+             is negative or larger than maxSize. Only the vint is read.
+   */
+  public static String readStringSafely(DataInput in,
+                                        int maxLength
+                                        ) throws IOException, 
+                                                 IllegalArgumentException {
+    int length = readVInt(in);
+    if (length < 0 || length > maxLength) {
+      throw new IllegalArgumentException("Encoded byte size for String was " + length + 
+                                         ", which is outside of 0.." +
+                                         maxLength + " range.");
+    }
+    byte [] bytes = new byte[length];
+    in.readFully(bytes, 0, length);
+    return Text.decode(bytes);
+  }
 }



Mime
View raw message