hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1441079 - in /hadoop/common/branches/branch-1-win: CHANGES.branch-1-win.txt src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
Date Thu, 31 Jan 2013 17:27:56 GMT
Author: suresh
Date: Thu Jan 31 17:27:56 2013
New Revision: 1441079

URL: http://svn.apache.org/viewvc?rev=1441079&view=rev
Log:
MAPREDUCE-4969. Merge change 1441078 from branch-1

Modified:
    hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
    hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java

Modified: hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt?rev=1441079&r1=1441078&r2=1441079&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt (original)
+++ hadoop/common/branches/branch-1-win/CHANGES.branch-1-win.txt Thu Jan 31 17:27:56 2013
@@ -340,3 +340,6 @@ Branch-hadoop-1-win (branched from branc
 
     HADOOP-9177. Address issues that reported by static code analysis on
     winutils. (Ivan Mitic via suresh)
+
+    MAPREDUCE-4969. TestKeyValueTextInputFormat test fails with Open JDK 7.
+    (Arpit Agarwal via suresh)

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java?rev=1441079&r1=1441078&r2=1441079&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
Thu Jan 31 17:27:56 2013
@@ -47,14 +47,10 @@ public class TestKeyValueTextInputFormat
   private static Path workDir = 
     new Path(new Path(System.getProperty("test.build.data", "."), "data"),
              "TestKeyValueTextInputFormat");
-
-  private static Path workDirFormat =
-      new Path(new Path(System.getProperty("test.build.data", "."), "data"),
-          "TestKeyValueTextInputFormat-testFormat");
   
   public void testFormat() throws Exception {
     JobConf job = new JobConf();
-    Path file = new Path(workDirFormat, "test.txt");
+    Path file = new Path(workDir, "test.txt");
 
     // A reporter that does nothing
     Reporter reporter = Reporter.NULL;
@@ -63,8 +59,8 @@ public class TestKeyValueTextInputFormat
     LOG.info("seed = "+seed);
     Random random = new Random(seed);
 
-    localFs.delete(workDirFormat, true);
-    FileInputFormat.setInputPaths(job, workDirFormat);
+    localFs.delete(workDir, true);
+    FileInputFormat.setInputPaths(job, workDir);
 
     // for a variety of lengths
     for (int length = 0; length < MAX_LENGTH;
@@ -140,32 +136,47 @@ public class TestKeyValueTextInputFormat
   }
   
   public void testUTF8() throws Exception {
-    LineReader in = makeStream("abcd\u20acbdcd\u20ac");
-    Text line = new Text();
-    in.readLine(line);
-    assertEquals("readLine changed utf8 characters", 
-                 "abcd\u20acbdcd\u20ac", line.toString());
-    in = makeStream("abc\u200axyz");
-    in.readLine(line);
-    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+    LineReader in = null;
+
+    try {
+      in = makeStream("abcd\u20acbdcd\u20ac");
+      Text line = new Text();
+      in.readLine(line);
+      assertEquals("readLine changed utf8 characters",
+                   "abcd\u20acbdcd\u20ac", line.toString());
+      in = makeStream("abc\u200axyz");
+      in.readLine(line);
+      assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+    } finally {
+      if (in != null) {
+        in.close();
+      }
+    }
   }
 
   public void testNewLines() throws Exception {
-    LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
-    Text out = new Text();
-    in.readLine(out);
-    assertEquals("line1 length", 1, out.getLength());
-    in.readLine(out);
-    assertEquals("line2 length", 2, out.getLength());
-    in.readLine(out);
-    assertEquals("line3 length", 0, out.getLength());
-    in.readLine(out);
-    assertEquals("line4 length", 3, out.getLength());
-    in.readLine(out);
-    assertEquals("line5 length", 4, out.getLength());
-    in.readLine(out);
-    assertEquals("line5 length", 5, out.getLength());
-    assertEquals("end of file", 0, in.readLine(out));
+    LineReader in = null;
+    try {
+      in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
+      Text out = new Text();
+      in.readLine(out);
+      assertEquals("line1 length", 1, out.getLength());
+      in.readLine(out);
+      assertEquals("line2 length", 2, out.getLength());
+      in.readLine(out);
+      assertEquals("line3 length", 0, out.getLength());
+      in.readLine(out);
+      assertEquals("line4 length", 3, out.getLength());
+      in.readLine(out);
+      assertEquals("line5 length", 4, out.getLength());
+      in.readLine(out);
+      assertEquals("line5 length", 5, out.getLength());
+      assertEquals("end of file", 0, in.readLine(out));
+    } finally {
+      if (in != null) {
+        in.close();
+      }
+    }
   }
   
   private static void writeFile(FileSystem fs, Path name, 
@@ -187,15 +198,23 @@ public class TestKeyValueTextInputFormat
                                       InputSplit split, 
                                       JobConf job) throws IOException {
     List<Text> result = new ArrayList<Text>();
-    RecordReader<Text, Text> reader = format.getRecordReader(split, job,
-                                                 voidReporter);
-    Text key = reader.createKey();
-    Text value = reader.createValue();
-    while (reader.next(key, value)) {
-      result.add(value);
-      value = (Text) reader.createValue();
+    RecordReader<Text, Text> reader = null;
+
+    try {
+      reader = format.getRecordReader(split, job, voidReporter);
+      Text key = reader.createKey();
+      Text value = reader.createValue();
+      while (reader.next(key, value)) {
+        result.add(value);
+        value = (Text) reader.createValue();
+      }
+
+      return result;
+    } finally {
+      if (reader != null) {
+        reader.close();
+      }
     }
-    return result;
   }
   
   /**

Modified: hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java?rev=1441079&r1=1441078&r2=1441079&view=diff
==============================================================================
--- hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
(original)
+++ hadoop/common/branches/branch-1-win/src/test/org/apache/hadoop/mapreduce/lib/input/TestKeyValueTextInputFormat.java
Thu Jan 31 17:27:56 2013
@@ -56,20 +56,16 @@ public class TestKeyValueTextInputFormat
     new Path(new Path(System.getProperty("test.build.data", "."), "data"),
              "TestKeyValueTextInputFormat");
 
-  private static Path workDirFormat =
-      new Path(new Path(System.getProperty("test.build.data", "."), "data"),
-          "TestKeyValueTextInputFormat-testFormat");
-  
   public void testFormat() throws Exception {
     Job job = new Job(defaultConf);
-    Path file = new Path(workDirFormat, "test.txt");
+    Path file = new Path(workDir, "test.txt");
 
     int seed = new Random().nextInt();
     LOG.info("seed = "+seed);
     Random random = new Random(seed);
 
-    localFs.delete(workDirFormat, true);
-    FileInputFormat.setInputPaths(job, workDirFormat);
+    localFs.delete(workDir, true);
+    FileInputFormat.setInputPaths(job, workDir);
 
     // for a variety of lengths
     for (int length = 0; length < MAX_LENGTH;
@@ -137,32 +133,46 @@ public class TestKeyValueTextInputFormat
   }
   
   public void testUTF8() throws Exception {
-    LineReader in = makeStream("abcd\u20acbdcd\u20ac");
-    Text line = new Text();
-    in.readLine(line);
-    assertEquals("readLine changed utf8 characters", 
-                 "abcd\u20acbdcd\u20ac", line.toString());
-    in = makeStream("abc\u200axyz");
-    in.readLine(line);
-    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+    LineReader in = null;
+    try {
+      in = makeStream("abcd\u20acbdcd\u20ac");
+      Text line = new Text();
+      in.readLine(line);
+      assertEquals("readLine changed utf8 characters", 
+                   "abcd\u20acbdcd\u20ac", line.toString());
+      in = makeStream("abc\u200axyz");
+      in.readLine(line);
+      assertEquals("split on fake newline", "abc\u200axyz", line.toString());
+    } finally {
+      if (in != null) {
+        in.close();
+      }
+    }
   }
 
   public void testNewLines() throws Exception {
-    LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
-    Text out = new Text();
-    in.readLine(out);
-    assertEquals("line1 length", 1, out.getLength());
-    in.readLine(out);
-    assertEquals("line2 length", 2, out.getLength());
-    in.readLine(out);
-    assertEquals("line3 length", 0, out.getLength());
-    in.readLine(out);
-    assertEquals("line4 length", 3, out.getLength());
-    in.readLine(out);
-    assertEquals("line5 length", 4, out.getLength());
-    in.readLine(out);
-    assertEquals("line5 length", 5, out.getLength());
-    assertEquals("end of file", 0, in.readLine(out));
+    LineReader in = null;
+    try {
+      in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
+      Text out = new Text();
+      in.readLine(out);
+      assertEquals("line1 length", 1, out.getLength());
+      in.readLine(out);
+      assertEquals("line2 length", 2, out.getLength());
+      in.readLine(out);
+      assertEquals("line3 length", 0, out.getLength());
+      in.readLine(out);
+      assertEquals("line4 length", 3, out.getLength());
+      in.readLine(out);
+      assertEquals("line5 length", 4, out.getLength());
+      in.readLine(out);
+      assertEquals("line5 length", 5, out.getLength());
+      assertEquals("end of file", 0, in.readLine(out));
+    } finally {
+      if (in != null) {
+        in.close();
+      }
+    }
   }
   
   private static void writeFile(FileSystem fs, Path name, 
@@ -182,12 +192,19 @@ public class TestKeyValueTextInputFormat
                                       InputSplit split, 
                                       TaskAttemptContext context) throws IOException, InterruptedException
{
     List<Text> result = new ArrayList<Text>();
-    RecordReader<Text, Text> reader = format.createRecordReader(split, context);
-    reader.initialize(split, context);
-    while (reader.nextKeyValue()) {
-      result.add(new Text(reader.getCurrentValue()));
+    RecordReader<Text, Text> reader = null;
+    try {
+      reader = format.createRecordReader(split, context);
+      reader.initialize(split, context);
+      while (reader.nextKeyValue()) {
+        result.add(new Text(reader.getCurrentValue()));
+      }
+      return result;
+    } finally {
+      if (reader != null) {
+        reader.close();
+      }
     }
-    return result;
   }
   
   /**



Mime
View raw message