hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1570694 - in /hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common: ./ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/fs/permission/ src/main/java/org/apache/hadoop/fs/shell/ src/main/jav...
Date Fri, 21 Feb 2014 20:08:09 GMT
Author: szetszwo
Date: Fri Feb 21 20:08:08 2014
New Revision: 1570694

URL: http://svn.apache.org/r1570694
Log:
Merge r1569890 through r1570692 from trunk.

Added:
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestProgress.java
      - copied unchanged from r1570692, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestProgress.java
Modified:
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt   (contents,
props changed)
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/  
(props changed)
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
    hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java

Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt Fri Feb
21 20:08:08 2014
@@ -326,6 +326,11 @@ Trunk (Unreleased)
     HADOOP-10344. Fix TestAclCommands after merging HADOOP-10338 patch.
     (cnauroth)
 
+    HADOOP-10352. Recursive setfacl erroneously attempts to apply default ACL to
+    files. (cnauroth)
+
+    HADOOP-10354. TestWebHDFS fails after merge of HDFS-4685 to trunk. (cnauroth)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -396,6 +401,8 @@ Release 2.4.0 - UNRELEASED
     HADOOP-10328. loadGenerator exit code is not reliable.
     (Haohui Mai via cnauroth)
 
+    HADOOP-10355. Fix TestLoadGenerator#testLoadGenerator. (Haohui Mai via jing9)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:r1570392-1570692

Propchange: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:r1570392-1570692

Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
Fri Feb 21 20:08:08 2014
@@ -569,6 +569,11 @@ public class RawLocalFileSystem extends 
         //expected format
         //-rw-------    1 username groupname ...
         String permission = t.nextToken();
+        if (permission.length() > FsPermission.MAX_PERMISSION_LENGTH) {
+          //files with ACLs might have a '+'
+          permission = permission.substring(0,
+            FsPermission.MAX_PERMISSION_LENGTH);
+        }
         setPermission(FsPermission.valueOf(permission));
         t.nextToken();
 

Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
Fri Feb 21 20:08:08 2014
@@ -48,6 +48,9 @@ public class FsPermission implements Wri
     WritableFactories.setFactory(ImmutableFsPermission.class, FACTORY);
   }
 
+  /** Maximum acceptable length of a permission string to parse */
+  public static final int MAX_PERMISSION_LENGTH = 10;
+
   /** Create an immutable {@link FsPermission} object. */
   public static FsPermission createImmutable(short permission) {
     return new ImmutableFsPermission(permission);
@@ -319,9 +322,10 @@ public class FsPermission implements Wri
     if (unixSymbolicPermission == null) {
       return null;
     }
-    else if (unixSymbolicPermission.length() != 10) {
-      throw new IllegalArgumentException("length != 10(unixSymbolicPermission="
-          + unixSymbolicPermission + ")");
+    else if (unixSymbolicPermission.length() != MAX_PERMISSION_LENGTH) {
+      throw new IllegalArgumentException(String.format(
+        "length != %d(unixSymbolicPermission=%s)", MAX_PERMISSION_LENGTH,
+        unixSymbolicPermission));
     }
 
     int n = 0;

Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
Fri Feb 21 20:08:08 2014
@@ -22,6 +22,8 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 
+import com.google.common.collect.Lists;
+
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -231,6 +233,7 @@ class AclCommands extends FsCommand {
     CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "b", "k", "R",
         "m", "x", "-set");
     List<AclEntry> aclEntries = null;
+    List<AclEntry> accessAclEntries = null;
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
@@ -263,6 +266,19 @@ class AclCommands extends FsCommand {
       if (args.size() > 1) {
         throw new HadoopIllegalArgumentException("Too many arguments");
       }
+
+      // In recursive mode, save a separate list of just the access ACL entries.
+      // Only directories may have a default ACL.  When a recursive operation
+      // encounters a file under the specified path, it must pass only the
+      // access ACL entries.
+      if (isRecursive() && (oneModifyOption || setOption)) {
+        accessAclEntries = Lists.newArrayList();
+        for (AclEntry entry: aclEntries) {
+          if (entry.getScope() == AclEntryScope.ACCESS) {
+            accessAclEntries.add(entry);
+          }
+        }
+      }
     }
 
     @Override
@@ -272,11 +288,37 @@ class AclCommands extends FsCommand {
       } else if (cf.getOpt("k")) {
         item.fs.removeDefaultAcl(item.path);
       } else if (cf.getOpt("m")) {
-        item.fs.modifyAclEntries(item.path, aclEntries);
+        List<AclEntry> entries = getAclEntries(item);
+        if (!entries.isEmpty()) {
+          item.fs.modifyAclEntries(item.path, entries);
+        }
       } else if (cf.getOpt("x")) {
-        item.fs.removeAclEntries(item.path, aclEntries);
+        List<AclEntry> entries = getAclEntries(item);
+        if (!entries.isEmpty()) {
+          item.fs.removeAclEntries(item.path, entries);
+        }
       } else if (cf.getOpt("-set")) {
-        item.fs.setAcl(item.path, aclEntries);
+        List<AclEntry> entries = getAclEntries(item);
+        if (!entries.isEmpty()) {
+          item.fs.setAcl(item.path, entries);
+        }
+      }
+    }
+
+    /**
+     * Returns the ACL entries to use in the API call for the given path.  For a
+     * recursive operation, returns all specified ACL entries if the item is a
+     * directory or just the access ACL entries if the item is a file.  For a
+     * non-recursive operation, returns all specified ACL entries.
+     *
+     * @param item PathData path to check
+     * @return List<AclEntry> ACL entries to use in the API call
+     */
+    private List<AclEntry> getAclEntries(PathData item) {
+      if (isRecursive()) {
+        return item.stat.isDirectory() ? aclEntries : accessAclEntries;
+      } else {
+        return aclEntries;
       }
     }
   }

Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
Fri Feb 21 20:08:08 2014
@@ -160,6 +160,31 @@ public class Progress {
 
   /** Called during execution on a leaf node to set its progress. */
   public synchronized void set(float progress) {
+    if (Float.isNaN(progress)) {
+      progress = 0;
+      LOG.warn("Illegal progress value found, progress is Float.NaN. " +
+        "Progress will be changed to 0");
+    }
+    else if (progress == Float.NEGATIVE_INFINITY) {
+      progress = 0;
+      LOG.warn("Illegal progress value found, progress is " +
+        "Float.NEGATIVE_INFINITY. Progress will be changed to 0");
+    }
+    else if (progress < 0) {
+      progress = 0;
+      LOG.warn("Illegal progress value found, progress is less than 0." +
+        " Progress will be changed to 0");
+    }
+    else if (progress > 1) {
+      progress = 1;
+      LOG.warn("Illegal progress value found, progress is larger than 1." +
+        " Progress will be changed to 1");
+    }
+    else if (progress == Float.POSITIVE_INFINITY) {
+      progress = 1;
+      LOG.warn("Illegal progress value found, progress is " +
+        "Float.POSITIVE_INFINITY. Progress will be changed to 1");
+    }
     this.progress = progress;
   }
 

Modified: hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java?rev=1570694&r1=1570693&r2=1570694&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
(original)
+++ hadoop/common/branches/HDFS-5535/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java
Fri Feb 21 20:08:08 2014
@@ -45,6 +45,8 @@ import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Preconditions;
+
 /** The load generator is a tool for testing NameNode behavior under
  * different client loads.
  * It allows the user to generate different mixes of read, write,
@@ -488,7 +490,35 @@ public class LoadGenerator extends Confi
     
     return initFileDirTables();
   }
-  
+
+  private static void parseScriptLine(String line, ArrayList<Long> duration,
+      ArrayList<Double> readProb, ArrayList<Double> writeProb) {
+    String[] a = line.split("\\s");
+
+    if (a.length != 3) {
+      throw new IllegalArgumentException("Incorrect number of parameters: "
+          + line);
+    }
+
+    try {
+      long d = Long.parseLong(a[0]);
+      double r = Double.parseDouble(a[1]);
+      double w = Double.parseDouble(a[2]);
+
+      Preconditions.checkArgument(d >= 0, "Invalid duration: " + d);
+      Preconditions.checkArgument(0 <= r && r <= 1.0,
+          "The read probability must be [0, 1]: " + r);
+      Preconditions.checkArgument(0 <= w && w <= 1.0,
+          "The read probability must be [0, 1]: " + w);
+
+      readProb.add(r);
+      duration.add(d);
+      writeProb.add(w);
+    } catch (NumberFormatException nfe) {
+      throw new IllegalArgumentException("Cannot parse: " + line);
+    }
+  }
+
   /**
    * Read a script file of the form: lines of text with duration in seconds,
    * read probability and write probability, separated by white space.
@@ -508,47 +538,19 @@ public class LoadGenerator extends Confi
     String line;
     // Read script, parse values, build array of duration, read and write probs
 
-    while ((line = br.readLine()) != null) {
-      lineNum++;
-      if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
-        continue;
-
-      String[] a = line.split("\\s");
-      if (a.length != 3) {
-        System.err.println("Line " + lineNum
-            + ": Incorrect number of parameters: " + line);
-      }
-
-      try {
-        long d = Long.parseLong(a[0]);
-        if (d < 0) {
-          System.err.println("Line " + lineNum + ": Invalid duration: " + d);
-          return -1;
-        }
-
-        double r = Double.parseDouble(a[1]);
-        if (r < 0.0 || r > 1.0) {
-          System.err.println("Line " + lineNum
-              + ": The read probability must be [0, 1]: " + r);
-          return -1;
-        }
-
-        double w = Double.parseDouble(a[2]);
-        if (w < 0.0 || w > 1.0) {
-          System.err.println("Line " + lineNum
-              + ": The read probability must be [0, 1]: " + r);
-          return -1;
-        }
+    try {
+      while ((line = br.readLine()) != null) {
+        lineNum++;
+        if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
+          continue;
 
-        readProb.add(r);
-        duration.add(d);
-        writeProb.add(w);
-      } catch (NumberFormatException nfe) {
-        System.err.println(lineNum + ": Can't parse: " + line);
-        return -1;
-      } finally {
-        IOUtils.cleanup(LOG, br);
+        parseScriptLine(line, duration, readProb, writeProb);
       }
+    } catch (IllegalArgumentException e) {
+      System.err.println("Line: " + lineNum + ", " + e.getMessage());
+      return -1;
+    } finally {
+      IOUtils.cleanup(LOG, br);
     }
     
     // Copy vectors to arrays of values, to avoid autoboxing overhead later



Mime
View raw message