hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hair...@apache.org
Subject svn commit: r1038003 - in /hadoop/common/trunk: ./ src/java/org/apache/hadoop/fs/ src/test/core/org/apache/hadoop/fs/
Date Tue, 23 Nov 2010 07:15:50 GMT
Author: hairong
Date: Tue Nov 23 07:15:49 2010
New Revision: 1038003

URL: http://svn.apache.org/viewvc?rev=1038003&view=rev
Log:
HADOOP-7023. Add listCorruptFileBlocks to FileSystem. Contributed by Patrick Kling.

Added:
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
    hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
Modified:
    hadoop/common/trunk/CHANGES.txt
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
    hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java

Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Tue Nov 23 07:15:49 2010
@@ -6,6 +6,9 @@ Trunk (unreleased changes)
 
   NEW FEATURES
 
+    HADOOP-7023. Add listCorruptFileBlocks to Filesysem. (Patrick Kling
+    via hairong)
+
   IMPROVEMENTS
 
     HADOOP-7042. Updates to test-patch.sh to include failed test names and

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java Tue Nov 23 07:15:49
2010
@@ -835,6 +835,18 @@ public abstract class AbstractFileSystem
       UnresolvedLinkException, IOException;
 
   /**
+   * @return a list in which each entry describes a corrupt file/block
+   * @throws IOException
+   */
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    throw new UnsupportedOperationException(getClass().getCanonicalName() +
+                                            " does not support" +
+                                            " listCorruptFileBlocks");
+  }
+
+  /**
    * The specification of this method matches that of
    * {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
    * must be for this file system.

Added: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java?rev=1038003&view=auto
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java (added)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CorruptFileBlocks.java Tue Nov 23 07:15:49
2010
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.Text;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Contains a list of paths corresponding to corrupt files and a cookie
+ * used for iterative calls to NameNode.listCorruptFileBlocks.
+ *
+ */
+public class CorruptFileBlocks implements Writable {
+  // used for hashCode
+  private static final int PRIME = 16777619;
+
+  private String[] files;
+  private String cookie;
+
+  public CorruptFileBlocks() {
+    this(new String[0], "");
+  }
+
+  public CorruptFileBlocks(String[] files, String cookie) {
+    this.files = files;
+    this.cookie = cookie;
+  }
+
+  public String[] getFiles() {
+    return files;
+  }
+
+  public String getCookie() {
+    return cookie;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    int fileCount = in.readInt();
+    files = new String[fileCount];
+    for (int i = 0; i < fileCount; i++) {
+      files[i] = Text.readString(in);
+    }
+    cookie = Text.readString(in);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(files.length);
+    for (int i = 0; i < files.length; i++) {
+      Text.writeString(out, files[i]);
+    }
+    Text.writeString(out, cookie);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof CorruptFileBlocks)) {
+      return false;
+    }
+    CorruptFileBlocks other = (CorruptFileBlocks) obj;
+    return cookie.equals(other.cookie) &&
+      Arrays.equals(files, other.files);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public int hashCode() {
+    int result = cookie.hashCode();
+
+    for (String file : files) {
+      result = PRIME * result + file.hashCode();
+    }
+
+    return result;
+  }
+}

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java Tue Nov 23 07:15:49
2010
@@ -1296,6 +1296,23 @@ public final class FileContext {
       }
     }.resolve(this, absF);
   }
+
+  /**
+   * @return a list in which each entry describes a corrupt file/block
+   * @throws IOException
+   */
+  public CorruptFileBlocks listCorruptFileBlocks(final String path,
+                                                 final String cookie)
+    throws IOException {
+    final Path absF = fixRelativePart(new Path(path));
+    return new FSLinkResolver<CorruptFileBlocks>() {
+      @Override
+      public CorruptFileBlocks next(final AbstractFileSystem fs, final Path p) 
+        throws IOException, UnresolvedLinkException {
+        return fs.listCorruptFileBlocks(p.toUri().getPath(), cookie);
+      }
+    }.resolve(this, absF);
+  }
   
   /**
    * List the statuses of the files/directories in the given path if the path is

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Tue Nov 23 07:15:49
2010
@@ -1091,6 +1091,18 @@ public abstract class FileSystem extends
   }
 
   /**
+   * @return a list in which each entry describes a corrupt file/block
+   * @throws IOException
+   */
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    throw new UnsupportedOperationException(getClass().getCanonicalName() +
+                                            " does not support" +
+                                            " listCorruptFileBlocks");
+  }
+
+  /**
    * Filter files/directories in the given path using the user-supplied path
    * filter.
    * 

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFileSystem.java Tue Nov 23 07:15:49
2010
@@ -165,7 +165,17 @@ public class FilterFileSystem extends Fi
   public FileStatus[] listStatus(Path f) throws IOException {
     return fs.listStatus(f);
   }
-  
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    return fs.listCorruptFileBlocks(path, cookie);
+  }
+
   /** List files and its block locations in a directory. */
   public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
   throws IOException {

Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java?rev=1038003&r1=1038002&r2=1038003&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java Tue Nov 23 07:15:49 2010
@@ -164,6 +164,16 @@ public abstract class FilterFs extends A
     return myFs.listStatus(f);
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    return myFs.listCorruptFileBlocks(path, cookie);
+  }
+
   @Override
   public void mkdir(Path dir, FsPermission permission, boolean createParent)
     throws IOException, UnresolvedLinkException {

Added: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java?rev=1038003&view=auto
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java (added)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java Tue
Nov 23 07:15:49 2010
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+
+public class TestCorruptFileBlocks {
+
+  /**
+   * Serialize the cfb given, deserialize and return the result.
+   */
+  static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb) 
+    throws IOException {
+    DataOutputBuffer buf = new DataOutputBuffer();
+    cfb.write(buf);
+
+    byte[] data = buf.getData();
+    DataInputStream input = new DataInputStream(new ByteArrayInputStream(data));
+
+    CorruptFileBlocks result = new CorruptFileBlocks();
+    result.readFields(input);
+
+    return result;
+  }
+
+  /**
+   * Check whether cfb is unchanged after serialization and deserialization.
+   */
+  static boolean checkSerialize(CorruptFileBlocks cfb)
+    throws IOException {
+    return cfb.equals(serializeAndDeserialize(cfb));
+  }
+
+  /**
+   * Test serialization and deserializaton of CorruptFileBlocks.
+   */
+  @Test
+  public void testSerialization() throws IOException {
+    {
+      CorruptFileBlocks cfb = new CorruptFileBlocks();
+      assertTrue(checkSerialize(cfb));
+    }
+
+    {
+      String[] files = new String[0];
+      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
+      assertTrue(checkSerialize(cfb));
+    }
+
+    {
+      String[] files = { "a", "bb", "ccc" };
+      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
+      assertTrue(checkSerialize(cfb));
+    }
+  }
+}
\ No newline at end of file



Mime
View raw message