hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hair...@apache.org
Subject svn commit: r1044146 - in /hadoop/common/trunk: ./ src/java/org/apache/hadoop/fs/ src/test/core/org/apache/hadoop/fs/
Date Thu, 09 Dec 2010 22:13:07 GMT
Author: hairong
Date: Thu Dec  9 22:13:06 2010
New Revision: 1044146

URL: http://svn.apache.org/viewvc?rev=1044146&view=rev
Log:
HADOOP-7060. A more elegant FileSystem#listCorruptFileBlocks API. Contributed by Patrick Kling.

Modified:
    hadoop/common/trunk/CHANGES.txt
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
    hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java

Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Thu Dec  9 22:13:06 2010
@@ -24,7 +24,10 @@ Trunk (unreleased changes)
     (Patrick Kling via eli)
 
     HADOOP-7054 Change NN LoadGenerator to use FileContext APIs
-	(Sanjay Radia)
+	  (Sanjay Radia)
+
+    HADOOP-7060. A more elegant FileSystem#listCorruptFileBlocks API.
+    (Patrick Kling via hairong)
 
   OPTIMIZATIONS
 

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java Thu Dec  9 22:13:06
2010
@@ -835,11 +835,11 @@ public abstract class AbstractFileSystem
       UnresolvedLinkException, IOException;
 
   /**
-   * @return a list in which each entry describes a corrupt file/block
+   * @return an iterator over the corrupt files under the given path
+   * (may contain duplicates if a file has more than one corrupt block)
    * @throws IOException
    */
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
-                                                 String cookie)
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
     throws IOException {
     throw new UnsupportedOperationException(getClass().getCanonicalName() +
                                             " does not support" +

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java Thu Dec  9 22:13:06
2010
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.Text;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Contains a list of paths corresponding to corrupt files and a cookie
- * used for iterative calls to NameNode.listCorruptFileBlocks.
- *
- */
-public class CorruptFileBlocks implements Writable {
-  // used for hashCode
-  private static final int PRIME = 16777619;
-
-  private String[] files;
-  private String cookie;
-
-  public CorruptFileBlocks() {
-    this(new String[0], "");
-  }
-
-  public CorruptFileBlocks(String[] files, String cookie) {
-    this.files = files;
-    this.cookie = cookie;
-  }
-
-  public String[] getFiles() {
-    return files;
-  }
-
-  public String getCookie() {
-    return cookie;
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    int fileCount = in.readInt();
-    files = new String[fileCount];
-    for (int i = 0; i < fileCount; i++) {
-      files[i] = Text.readString(in);
-    }
-    cookie = Text.readString(in);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(files.length);
-    for (int i = 0; i < files.length; i++) {
-      Text.writeString(out, files[i]);
-    }
-    Text.writeString(out, cookie);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  public boolean equals(Object obj) {
-    if (this == obj) {
-      return true;
-    }
-    if (!(obj instanceof CorruptFileBlocks)) {
-      return false;
-    }
-    CorruptFileBlocks other = (CorruptFileBlocks) obj;
-    return cookie.equals(other.cookie) &&
-      Arrays.equals(files, other.files);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  public int hashCode() {
-    int result = cookie.hashCode();
-
-    for (String file : files) {
-      result = PRIME * result + file.hashCode();
-    }
-
-    return result;
-  }
-}

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java Thu Dec  9 22:13:06
2010
@@ -1298,18 +1298,19 @@ public final class FileContext {
   }
 
   /**
-   * @return a list in which each entry describes a corrupt file/block
+   * @return an iterator over the corrupt files under the given path
+   * (may contain duplicates if a file has more than one corrupt block)
    * @throws IOException
    */
-  public CorruptFileBlocks listCorruptFileBlocks(final String path,
-                                                 final String cookie)
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
     throws IOException {
-    final Path absF = fixRelativePart(new Path(path));
-    return new FSLinkResolver<CorruptFileBlocks>() {
+    final Path absF = fixRelativePart(path);
+    return new FSLinkResolver<RemoteIterator<Path>>() {
       @Override
-      public CorruptFileBlocks next(final AbstractFileSystem fs, final Path p) 
+      public RemoteIterator<Path> next(final AbstractFileSystem fs,
+                                       final Path p) 
         throws IOException, UnresolvedLinkException {
-        return fs.listCorruptFileBlocks(p.toUri().getPath(), cookie);
+        return fs.listCorruptFileBlocks(p);
       }
     }.resolve(this, absF);
   }

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Thu Dec  9 22:13:06
2010
@@ -1091,11 +1091,11 @@ public abstract class FileSystem extends
   }
 
   /**
-   * @return a list in which each entry describes a corrupt file/block
+   * @return an iterator over the corrupt files under the given path
+   * (may contain duplicates if a file has more than one corrupt block)
    * @throws IOException
    */
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
-                                                 String cookie)
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
     throws IOException {
     throw new UnsupportedOperationException(getClass().getCanonicalName() +
                                             " does not support" +

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Thu Dec  9 22:13:06
2010
@@ -170,10 +170,9 @@ public class FilterFileSystem extends Fi
    * {@inheritDoc}
    */
   @Override
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
-                                                 String cookie)
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
     throws IOException {
-    return fs.listCorruptFileBlocks(path, cookie);
+    return fs.listCorruptFileBlocks(path);
   }
 
   /** List files and its block locations in a directory. */

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java Thu Dec  9 22:13:06 2010
@@ -168,10 +168,9 @@ public abstract class FilterFs extends A
    * {@inheritDoc}
    */
   @Override
-  public CorruptFileBlocks listCorruptFileBlocks(String path,
-                                                 String cookie)
+  public RemoteIterator<Path> listCorruptFileBlocks(Path path)
     throws IOException {
-    return myFs.listCorruptFileBlocks(path, cookie);
+    return myFs.listCorruptFileBlocks(path);
   }
 
   @Override

Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java?rev=1044146&r1=1044145&r2=1044146&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java Thu
Dec  9 22:13:06 2010
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.*;
-import org.junit.Test;
-
-import org.apache.hadoop.io.DataOutputBuffer;
-
-public class TestCorruptFileBlocks {
-
-  /**
-   * Serialize the cfb given, deserialize and return the result.
-   */
-  static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb) 
-    throws IOException {
-    DataOutputBuffer buf = new DataOutputBuffer();
-    cfb.write(buf);
-
-    byte[] data = buf.getData();
-    DataInputStream input = new DataInputStream(new ByteArrayInputStream(data));
-
-    CorruptFileBlocks result = new CorruptFileBlocks();
-    result.readFields(input);
-
-    return result;
-  }
-
-  /**
-   * Check whether cfb is unchanged after serialization and deserialization.
-   */
-  static boolean checkSerialize(CorruptFileBlocks cfb)
-    throws IOException {
-    return cfb.equals(serializeAndDeserialize(cfb));
-  }
-
-  /**
-   * Test serialization and deserializaton of CorruptFileBlocks.
-   */
-  @Test
-  public void testSerialization() throws IOException {
-    {
-      CorruptFileBlocks cfb = new CorruptFileBlocks();
-      assertTrue(checkSerialize(cfb));
-    }
-
-    {
-      String[] files = new String[0];
-      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
-      assertTrue(checkSerialize(cfb));
-    }
-
-    {
-      String[] files = { "a", "bb", "ccc" };
-      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
-      assertTrue(checkSerialize(cfb));
-    }
-  }
-}
\ No newline at end of file



Mime
View raw message