hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oz...@apache.org
Subject hadoop git commit: HADOOP-11283. SequenceFile.Writer can leak file descriptors in DistCpV1#setup(). (Contributed by Varun Saxena via ozawa)
Date Thu, 25 Dec 2014 14:31:21 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk b256dd760 -> a164ce2ac


HADOOP-11283. SequenceFile.Writer can leak file descriptors in DistCpV1#setup(). (Contributed
by Varun Saxena via ozawa)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a164ce2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a164ce2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a164ce2a

Branch: refs/heads/trunk
Commit: a164ce2ac985ecac957362fc717640ad45449371
Parents: b256dd7
Author: Tsuyoshi Ozawa <ozawa@apache.org>
Authored: Thu Dec 25 23:29:54 2014 +0900
Committer: Tsuyoshi Ozawa <ozawa@apache.org>
Committed: Thu Dec 25 23:30:45 2014 +0900

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +
 .../java/org/apache/hadoop/tools/DistCpV1.java  | 60 +++++++++-----------
 2 files changed, 30 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a164ce2a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0a04e40..02144fb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -650,6 +650,9 @@ Release 2.7.0 - UNRELEASED
     HADOOP-11414. FileBasedIPList#readLines() can leak file descriptors.
     (ozawa)
 
+    HADOOP-11283. SequenceFile.Writer can leak file descriptors in
+    DistCpV1#setup(). (Varun Saxena via ozawa)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a164ce2a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
index 6801d6f..c44b67b 100644
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
+++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
@@ -1258,42 +1258,40 @@ public class DistCpV1 implements Tool {
     FileSystem jobfs = jobDirectory.getFileSystem(jobConf);
 
     Path srcfilelist = new Path(jobDirectory, "_distcp_src_files");
-    jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
-    SequenceFile.Writer src_writer = SequenceFile.createWriter(jobfs, jobConf,
-        srcfilelist, LongWritable.class, FilePair.class,
-        SequenceFile.CompressionType.NONE);
-
     Path dstfilelist = new Path(jobDirectory, "_distcp_dst_files");
-    SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobfs, jobConf,
-        dstfilelist, Text.class, Text.class,
-        SequenceFile.CompressionType.NONE);
-
     Path dstdirlist = new Path(jobDirectory, "_distcp_dst_dirs");
+    jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
     jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString());
-    SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobfs, jobConf,
-        dstdirlist, Text.class, FilePair.class,
-        SequenceFile.CompressionType.NONE);
-
-    // handle the case where the destination directory doesn't exist
-    // and we've only a single src directory OR we're updating/overwriting
-    // the contents of the destination directory.
-    final boolean special =
-      (args.srcs.size() == 1 && !dstExists) || update || overwrite;
     int srcCount = 0, cnsyncf = 0, dirsyn = 0;
     long fileCount = 0L, dirCount = 0L, byteCount = 0L, cbsyncs = 0L,
          skipFileCount = 0L, skipByteCount = 0L;
-    
-    Path basedir = null;
-    HashSet<Path> parentDirsToCopy = new HashSet<Path>(); 
-    if (args.basedir != null) {
-      FileSystem basefs = args.basedir.getFileSystem(conf);
-      basedir = args.basedir.makeQualified(basefs);
-      if (!basefs.isDirectory(basedir)) {
-        throw new IOException("Basedir " + basedir + " is not a directory.");
+    try (
+        SequenceFile.Writer src_writer = SequenceFile.createWriter(jobfs,
+            jobConf, srcfilelist, LongWritable.class, FilePair.class,
+            SequenceFile.CompressionType.NONE);
+        SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobfs,
+            jobConf, dstfilelist, Text.class, Text.class,
+            SequenceFile.CompressionType.NONE);
+        SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobfs,
+            jobConf, dstdirlist, Text.class, FilePair.class,
+            SequenceFile.CompressionType.NONE)
+    ) {
+      // handle the case where the destination directory doesn't exist
+      // and we've only a single src directory OR we're updating/overwriting
+      // the contents of the destination directory.
+      final boolean special =
+        (args.srcs.size() == 1 && !dstExists) || update || overwrite;
+
+      Path basedir = null;
+      HashSet<Path> parentDirsToCopy = new HashSet<Path>();
+      if (args.basedir != null) {
+        FileSystem basefs = args.basedir.getFileSystem(conf);
+        basedir = args.basedir.makeQualified(basefs);
+        if (!basefs.isDirectory(basedir)) {
+          throw new IOException("Basedir " + basedir + " is not a directory.");
+        }
       }
-    }
-    
-    try {
+
       for(Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext(); ) {
         final Path src = srcItr.next();
         FileSystem srcfs = src.getFileSystem(conf);
@@ -1426,10 +1424,6 @@ public class DistCpV1 implements Tool {
           }
         }
       }
-    } finally {
-      checkAndClose(src_writer);
-      checkAndClose(dst_writer);
-      checkAndClose(dir_writer);
     }
     LOG.info("sourcePathsCount(files+directories)=" + srcCount);
     LOG.info("filesToCopyCount=" + fileCount);


Mime
View raw message