Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3F36918F9D for ; Tue, 2 Feb 2016 18:43:50 +0000 (UTC) Received: (qmail 54469 invoked by uid 500); 2 Feb 2016 18:43:34 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 54402 invoked by uid 500); 2 Feb 2016 18:43:34 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 54392 invoked by uid 99); 2 Feb 2016 18:43:34 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Feb 2016 18:43:34 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F0872DFD7D; Tue, 2 Feb 2016 18:43:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: lei@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains INodeReference. (Xiao Chen via lei) Date: Tue, 2 Feb 2016 18:43:33 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 700a176fa -> 9d494f0c0 HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains INodeReference. (Xiao Chen via lei) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d494f0c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d494f0c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d494f0c Branch: refs/heads/trunk Commit: 9d494f0c0eaa05417f3a3e88487d878d1731da36 Parents: 700a176 Author: Lei Xu Authored: Tue Feb 2 10:41:17 2016 -0800 Committer: Lei Xu Committed: Tue Feb 2 10:42:01 2016 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../tools/offlineImageViewer/FSImageLoader.java | 2 +- .../IgnoreSnapshotException.java | 28 ++++++ .../PBImageDelimitedTextWriter.java | 23 ++++- .../offlineImageViewer/PBImageTextWriter.java | 90 ++++++++++++++++---- .../TestOfflineImageViewer.java | 36 +++++--- 6 files changed, 153 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d494f0c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 366f330..0888a1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1872,6 +1872,9 @@ Release 2.8.0 - UNRELEASED HDFS-9638. Improve DistCp Help and documentation. (Wei-Chiu Chuang via Yongjun Zhang) + HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains + INodeReference. (Xiao Chen via lei) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d494f0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java index 4d8eb35..172f599 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java @@ -204,7 +204,7 @@ class FSImageLoader { return dirs; } - private static ImmutableList loadINodeReferenceSection(InputStream in) + static ImmutableList loadINodeReferenceSection(InputStream in) throws IOException { LOG.info("Loading inode references"); ImmutableList.Builder builder = ImmutableList.builder(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d494f0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java new file mode 100644 index 0000000..60a6534 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import java.io.IOException; + +/** + * Signals that a snapshot is ignored. + */ +public class IgnoreSnapshotException extends IOException { + public IgnoreSnapshotException() { + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d494f0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java index fbe7f3a..3ddc135 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java @@ -84,6 +84,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter { inodeName.isEmpty() ? "/" : inodeName); buffer.append(path.toString()); PermissionStatus p = null; + boolean isDir = false; switch (inode.getType()) { case FILE: @@ -109,6 +110,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter { append(buffer, 0); // Num bytes. append(buffer, dir.getNsQuota()); append(buffer, dir.getDsQuota()); + isDir = true; break; case SYMLINK: INodeSymlink s = inode.getSymlink(); @@ -126,9 +128,28 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter { break; } assert p != null; - append(buffer, p.getPermission().toString()); + String dirString = isDir ? "d" : "-"; + append(buffer, dirString + p.getPermission().toString()); append(buffer, p.getUserName()); append(buffer, p.getGroupName()); return buffer.toString(); } + + @Override + public String getHeader() { + StringBuffer buffer = new StringBuffer(); + buffer.append("Path"); + append(buffer, "Replication"); + append(buffer, "ModificationTime"); + append(buffer, "AccessTime"); + append(buffer, "PreferredBlockSize"); + append(buffer, "BlocksCount"); + append(buffer, "FileSize"); + append(buffer, "NSQUOTA"); + append(buffer, "DSQUOTA"); + append(buffer, "Permission"); + append(buffer, "UserName"); + append(buffer, "GroupName"); + return buffer.toString(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d494f0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java index d2ccc5c..ae1cd79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -195,13 +196,17 @@ abstract class PBImageTextWriter implements Closeable { dirMap.put(p.getId(), dir); } + @Override public String getParentPath(long inode) throws IOException { if (inode == INodeId.ROOT_INODE_ID) { return ""; } Dir parent = dirChildMap.get(inode); - Preconditions.checkState(parent != null, - "Can not find parent directory for INode: %s", inode); + if (parent == null) { + // The inode is an INodeReference, which is generated from snapshot. + // For delimited oiv tool, no need to print out metadata in snapshots. + PBImageTextWriter.ignoreSnapshotName(inode); + } return parent.getPath(); } @@ -353,16 +358,22 @@ abstract class PBImageTextWriter implements Closeable { return "/"; } byte[] bytes = dirChildMap.get(toBytes(inode)); - Preconditions.checkState(bytes != null && bytes.length == 8, - "Can not find parent directory for inode %s, " - + "fsimage might be corrupted", inode); + if (bytes == null) { + // The inode is an INodeReference, which is generated from snapshot. + // For delimited oiv tool, no need to print out metadata in snapshots. + PBImageTextWriter.ignoreSnapshotName(inode); + } + if (bytes.length != 8) { + throw new IOException( + "bytes array length error. Actual length is " + bytes.length); + } long parent = toLong(bytes); if (!dirPathCache.containsKey(parent)) { bytes = dirMap.get(toBytes(parent)); - if (parent != INodeId.ROOT_INODE_ID) { - Preconditions.checkState(bytes != null, - "Can not find parent directory for inode %s, " - + ", the fsimage might be corrupted.", parent); + if (parent != INodeId.ROOT_INODE_ID && bytes == null) { + // The parent is an INodeReference, which is generated from snapshot. + // For delimited oiv tool, no need to print out metadata in snapshots. + PBImageTextWriter.ignoreSnapshotName(parent); } String parentName = toString(bytes); String parentPath = @@ -401,6 +412,7 @@ abstract class PBImageTextWriter implements Closeable { @Override public void close() throws IOException { + out.flush(); IOUtils.cleanup(null, metadataMap); } @@ -411,6 +423,11 @@ abstract class PBImageTextWriter implements Closeable { */ abstract protected String getEntry(String parent, INode inode); + /** + * Get text output for the header line. + */ + abstract protected String getHeader(); + public void visit(RandomAccessFile file) throws IOException { Configuration conf = new Configuration(); if (!FSImageUtil.checkFileFormat(file)) { @@ -442,6 +459,7 @@ abstract class PBImageTextWriter implements Closeable { } }); + ImmutableList refIdList = null; for (FileSummary.Section section : sections) { fin.getChannel().position(section.getOffset()); is = FSImageUtil.wrapInputStreamForCompression(conf, @@ -449,15 +467,22 @@ abstract class PBImageTextWriter implements Closeable { fin, section.getLength()))); switch (SectionName.fromString(section.getName())) { case STRING_TABLE: + LOG.info("Loading string table"); stringTable = FSImageLoader.loadStringTable(is); break; + case INODE_REFERENCE: + // Load INodeReference so that all INodes can be processed. + // Snapshots are not handled and will just be ignored for now. + LOG.info("Loading inode references"); + refIdList = FSImageLoader.loadINodeReferenceSection(is); + break; default: break; } } loadDirectories(fin, sections, summary, conf); - loadINodeDirSection(fin, sections, summary, conf); + loadINodeDirSection(fin, sections, summary, conf, refIdList); metadataMap.sync(); output(conf, summary, fin, sections); } @@ -468,6 +493,7 @@ abstract class PBImageTextWriter implements Closeable { throws IOException { InputStream is; long startTime = Time.monotonicNow(); + out.println(getHeader()); for (FileSummary.Section section : sections) { if (SectionName.fromString(section.getName()) == SectionName.INODE) { fin.getChannel().position(section.getOffset()); @@ -508,7 +534,7 @@ abstract class PBImageTextWriter implements Closeable { private void loadINodeDirSection( FileInputStream fin, List sections, - FileSummary summary, Configuration conf) + FileSummary summary, Configuration conf, List refIdList) throws IOException { LOG.info("Loading INode directory section."); long startTime = Time.monotonicNow(); @@ -519,7 +545,7 @@ abstract class PBImageTextWriter implements Closeable { InputStream is = FSImageUtil.wrapInputStreamForCompression(conf, summary.getCodec(), new BufferedInputStream( new LimitInputStream(fin, section.getLength()))); - buildNamespace(is); + buildNamespace(is, refIdList); } } long timeTaken = Time.monotonicNow() - startTime; @@ -549,7 +575,8 @@ abstract class PBImageTextWriter implements Closeable { /** * Scan the INodeDirectory section to construct the namespace. */ - private void buildNamespace(InputStream in) throws IOException { + private void buildNamespace(InputStream in, List refIdList) + throws IOException { int count = 0; while (true) { FsImageProto.INodeDirectorySection.DirEntry e = @@ -562,12 +589,15 @@ abstract class PBImageTextWriter implements Closeable { LOG.debug("Scanned {} directories.", count); } long parentId = e.getParent(); - // Referred INode is not support for now. for (int i = 0; i < e.getChildrenCount(); i++) { long childId = e.getChildren(i); metadataMap.putDirChild(parentId, childId); } - Preconditions.checkState(e.getRefChildrenCount() == 0); + for (int i = e.getChildrenCount(); + i < e.getChildrenCount() + e.getRefChildrenCount(); i++) { + int refId = e.getRefChildren(i - e.getChildrenCount()); + metadataMap.putDirChild(parentId, refIdList.get(refId)); + } } LOG.info("Scanned {} INode directories to build namespace.", count); } @@ -575,15 +605,41 @@ abstract class PBImageTextWriter implements Closeable { private void outputINodes(InputStream in) throws IOException { INodeSection s = INodeSection.parseDelimitedFrom(in); LOG.info("Found {} INodes in the INode section", s.getNumInodes()); + long ignored = 0; + long ignoredSnapshots = 0; for (int i = 0; i < s.getNumInodes(); ++i) { INode p = INode.parseDelimitedFrom(in); - String parentPath = metadataMap.getParentPath(p.getId()); - out.println(getEntry(parentPath, p)); + try { + String parentPath = metadataMap.getParentPath(p.getId()); + out.println(getEntry(parentPath, p)); + } catch (IOException ioe) { + ignored++; + if (!(ioe instanceof IgnoreSnapshotException)) { + LOG.warn("Exception caught, ignoring node:{}", p.getId(), ioe); + } else { + ignoredSnapshots++; + if (LOG.isDebugEnabled()) { + LOG.debug("Exception caught, ignoring node:{}.", p.getId(), ioe); + } + } + } if (LOG.isDebugEnabled() && i % 100000 == 0) { LOG.debug("Outputted {} INodes.", i); } } + if (ignored > 0) { + LOG.warn("Ignored {} nodes, including {} in snapshots. Please turn on" + + " debug log for details", ignored, ignoredSnapshots); + } LOG.info("Outputted {} INodes.", s.getNumInodes()); } + + static void ignoreSnapshotName(long inode) throws IOException { + // Ignore snapshots - we want the output similar to -ls -R. + if (LOG.isDebugEnabled()) { + LOG.debug("No snapshot name found for inode {}", inode); + } + throw new IgnoreSnapshotException(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d494f0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 467534a..91b79e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -87,6 +87,7 @@ public class TestOfflineImageViewer { // namespace as written to dfs, to be compared with viewer's output final static HashMap writtenFiles = Maps.newHashMap(); + static int dirCount = 0; @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -113,7 +114,7 @@ public class TestOfflineImageViewer { DistributedFileSystem hdfs = cluster.getFileSystem(); // Create a reasonable namespace - for (int i = 0; i < NUM_DIRS; i++) { + for (int i = 0; i < NUM_DIRS; i++, dirCount++) { Path dir = new Path("/dir" + i); hdfs.mkdirs(dir); writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString())); @@ -131,11 +132,13 @@ public class TestOfflineImageViewer { // Create an empty directory Path emptydir = new Path("/emptydir"); hdfs.mkdirs(emptydir); + dirCount++; writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir)); //Create a directory whose name should be escaped in XML Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here"); hdfs.mkdirs(invalidXMLDir); + dirCount++; // Get delegation tokens so we log the delegation token op Token[] delegationTokens = hdfs @@ -144,15 +147,24 @@ public class TestOfflineImageViewer { LOG.debug("got token " + t); } - final Path snapshot = new Path("/snapshot"); - hdfs.mkdirs(snapshot); - hdfs.allowSnapshot(snapshot); - hdfs.mkdirs(new Path("/snapshot/1")); - hdfs.delete(snapshot, true); + // Create INodeReference + final Path src = new Path("/src"); + hdfs.mkdirs(src); + dirCount++; + writtenFiles.put(src.toString(), hdfs.getFileStatus(src)); + final Path orig = new Path("/src/orig"); + hdfs.mkdirs(orig); + hdfs.allowSnapshot(src); + hdfs.createSnapshot(src, "snapshot"); + final Path dst = new Path("/dst"); + hdfs.rename(orig, dst); + dirCount++; + writtenFiles.put(dst.toString(), hdfs.getFileStatus(dst)); // Set XAttrs so the fsimage contains XAttr ops final Path xattr = new Path("/xattr"); hdfs.mkdirs(xattr); + dirCount++; hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 }); hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 }); // OIV should be able to handle empty value XAttrs @@ -232,8 +244,8 @@ public class TestOfflineImageViewer { matcher = p.matcher(outputString); assertTrue(matcher.find() && matcher.groupCount() == 1); int totalDirs = Integer.parseInt(matcher.group(1)); - // totalDirs includes root directory, empty directory, and xattr directory - assertEquals(NUM_DIRS + 4, totalDirs); + // totalDirs includes root directory + assertEquals(dirCount + 1, totalDirs); FileStatus maxFile = Collections.max(writtenFiles.values(), new Comparator() { @@ -285,7 +297,7 @@ public class TestOfflineImageViewer { // verify the number of directories FileStatus[] statuses = webhdfs.listStatus(new Path("/")); - assertEquals(NUM_DIRS + 3, statuses.length); // contains empty and xattr directory + assertEquals(dirCount, statuses.length); // verify the number of files in the directory statuses = webhdfs.listStatus(new Path("/dir0")); @@ -393,11 +405,15 @@ public class TestOfflineImageViewer { BufferedReader reader = new BufferedReader(new InputStreamReader(input))) { String line; + boolean header = true; while ((line = reader.readLine()) != null) { System.out.println(line); String[] fields = line.split(DELIMITER); assertEquals(12, fields.length); - fileNames.add(fields[0]); + if (!header) { + fileNames.add(fields[0]); + } + header = false; } }