hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rohithsharm...@apache.org
Subject hadoop git commit: MAPREDUCE-6662. Clear ASF Warnings on test data files. Contributed by Vinayakumar B
Date Tue, 29 Mar 2016 02:36:06 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 edf17fe8e -> f22ce96b5


MAPREDUCE-6662. Clear ASF Warnings on test data files. Contributed by Vinayakumar B

(cherry picked from commit 0ef8bbfd8791899c7cfed3dd9c1670182fd87575)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f22ce96b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f22ce96b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f22ce96b

Branch: refs/heads/branch-2
Commit: f22ce96b50f8148e95a453ff41f84b99a9197bf6
Parents: edf17fe
Author: Rohith Sharma K S <rohithsharmaks@apache.org>
Authored: Tue Mar 29 10:32:21 2016 +0800
Committer: Rohith Sharma K S <rohithsharmaks@apache.org>
Committed: Tue Mar 29 10:35:04 2016 +0800

----------------------------------------------------------------------
 .../mapreduce/v2/hs/TestHistoryFileManager.java  |  5 +++--
 .../apache/hadoop/mapred/TestFieldSelection.java |  5 +++--
 .../mapred/TestMRTimelineEventHandling.java      | 19 ++++++++++++-------
 .../hadoop/mapred/TestMiniMRChildTask.java       | 16 ++++++++++------
 .../hadoop/mapred/TestOldCombinerGrouping.java   |  5 +++--
 .../mapred/lib/aggregate/TestAggregates.java     |  7 ++++---
 .../mapreduce/TestNewCombinerGrouping.java       |  5 +++--
 .../lib/aggregate/TestMapReduceAggregates.java   |  5 +++--
 .../lib/input/TestCombineFileInputFormat.java    |  4 +++-
 9 files changed, 44 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index 18e3e6a..aa2e979 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -71,8 +71,9 @@ public class TestHistoryFileManager {
     Configuration conf = new HdfsConfiguration();
     Configuration conf2 = new HdfsConfiguration();
     dfsCluster = new MiniDFSCluster.Builder(conf).build();
-    conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,
-            conf.get(MiniDFSCluster.HDFS_MINIDFS_BASEDIR) + "_2");
+    conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, conf.get(
+        MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MiniDFSCluster.getBaseDirectory())
+        + "_2");
     dfsCluster2 = new MiniDFSCluster.Builder(conf2).build();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
index 239c239..29333b7b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
@@ -44,8 +44,9 @@ private static NumberFormat idFormat = NumberFormat.getInstance();
     FileSystem fs = FileSystem.get(conf);
     int numOfInputLines = 10;
 
-    Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
-    Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
+    String baseDir = System.getProperty("test.build.data", "build/test/data");
+    Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test");
+    Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test");
     String inputFile = "input.txt";
     fs.delete(INPUT_DIR, true);
     fs.mkdirs(INPUT_DIR);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index a82007f..40ed9ad 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -97,9 +97,10 @@ public class TestMRTimelineEventHandling {
           + cluster.getApplicationHistoryServer().getPort());
       TimelineStore ts = cluster.getApplicationHistoryServer()
               .getTimelineStore();
-
-      Path inDir = new Path("input");
-      Path outDir = new Path("output");
+      String localPathRoot = System.getProperty("test.build.data",
+          "build/test/data");
+      Path inDir = new Path(localPathRoot, "input");
+      Path outDir = new Path(localPathRoot, "output");
       RunningJob job =
               UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
       Assert.assertEquals(JobStatus.SUCCEEDED,
@@ -155,8 +156,10 @@ public class TestMRTimelineEventHandling {
       TimelineStore ts = cluster.getApplicationHistoryServer()
           .getTimelineStore();
 
-      Path inDir = new Path("input");
-      Path outDir = new Path("output");
+      String localPathRoot = System.getProperty("test.build.data",
+          "build/test/data");
+      Path inDir = new Path(localPathRoot, "input");
+      Path outDir = new Path(localPathRoot, "output");
       RunningJob job =
           UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
       Assert.assertEquals(JobStatus.SUCCEEDED,
@@ -195,8 +198,10 @@ public class TestMRTimelineEventHandling {
       TimelineStore ts = cluster.getApplicationHistoryServer()
           .getTimelineStore();
 
-      Path inDir = new Path("input");
-      Path outDir = new Path("output");
+      String localPathRoot = System.getProperty("test.build.data",
+          "build/test/data");
+      Path inDir = new Path(localPathRoot, "input");
+      Path outDir = new Path(localPathRoot, "output");
 
       conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
       RunningJob job =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index 74be8a7..cbeeccf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -436,8 +436,10 @@ public class TestMiniMRChildTask {
 
       JobConf conf = new JobConf(mr.getConfig());
       // initialize input, output directories
-      Path inDir = new Path("input");
-      Path outDir = new Path("output");
+      Path rootDir = new Path(System.getProperty("test.build.data",
+          "build/test/data"));
+      Path inDir = new Path(rootDir, "input");
+      Path outDir = new Path(rootDir, "output");
       String input = "The input";
 
       // set config to use the ExecutionEnvCheckMapClass map class
@@ -462,9 +464,10 @@ public class TestMiniMRChildTask {
   public void testTaskEnv(){
     try {
       JobConf conf = new JobConf(mr.getConfig());
+      String baseDir = System.getProperty("test.build.data", "build/test/data");
       // initialize input, output directories
-      Path inDir = new Path("testing/wc/input1");
-      Path outDir = new Path("testing/wc/output1");
+      Path inDir = new Path(baseDir + "/testing/wc/input1");
+      Path outDir = new Path(baseDir + "/testing/wc/output1");
       FileSystem outFs = outDir.getFileSystem(conf);
       runTestTaskEnv(conf, inDir, outDir, false);
       outFs.delete(outDir, true);
@@ -485,9 +488,10 @@ public class TestMiniMRChildTask {
   public void testTaskOldEnv(){
     try {
       JobConf conf = new JobConf(mr.getConfig());
+      String baseDir = System.getProperty("test.build.data", "build/test/data");
       // initialize input, output directories
-      Path inDir = new Path("testing/wc/input1");
-      Path outDir = new Path("testing/wc/output1");
+      Path inDir = new Path(baseDir + "/testing/wc/input1");
+      Path outDir = new Path(baseDir + "/testing/wc/output1");
       FileSystem outFs = outDir.getFileSystem(conf);
       runTestTaskEnv(conf, inDir, outDir, true);
       outFs.delete(outDir, true);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
index 80a6702..046c2d3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
@@ -37,8 +37,9 @@ import java.util.Set;
 import java.util.UUID;
 
 public class TestOldCombinerGrouping {
-  private static String TEST_ROOT_DIR =
-      new File("build", UUID.randomUUID().toString()).getAbsolutePath();
+  private static String TEST_ROOT_DIR = new File(System.getProperty(
+      "test.build.data", "build/test/data"), UUID.randomUUID().toString())
+          .getAbsolutePath();
 
   public static class Map implements
       Mapper<LongWritable, Text, Text, LongWritable> {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
index 6da96ce..e2fdd42 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
@@ -46,8 +46,9 @@ public class TestAggregates extends TestCase {
     FileSystem fs = FileSystem.get(conf);
     int numOfInputLines = 20;
 
-    Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
-    Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
+    String baseDir = System.getProperty("test.build.data", "build/test/data");
+    Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test");
+    Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test");
     String inputFile = "input.txt";
     fs.delete(INPUT_DIR, true);
     fs.mkdirs(INPUT_DIR);
@@ -115,7 +116,7 @@ public class TestAggregates extends TestCase {
     outdata = outdata.substring(0, expectedOutput.toString().length());
 
     assertEquals(expectedOutput.toString(),outdata);
-    //fs.delete(OUTPUT_DIR);
+    fs.delete(OUTPUT_DIR, true);
     fs.delete(INPUT_DIR, true);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
index ab9ddbb..c2054f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
@@ -39,8 +39,9 @@ import java.util.Set;
 import java.util.UUID;
 
 public class TestNewCombinerGrouping {
-  private static String TEST_ROOT_DIR =
-      new File("build", UUID.randomUUID().toString()).getAbsolutePath();
+  private static String TEST_ROOT_DIR = new File(System.getProperty(
+      "test.build.data", "build/test/data"), UUID.randomUUID().toString())
+          .getAbsolutePath();
 
   public static class Map extends
       Mapper<LongWritable, Text, Text, LongWritable> {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
index f24dffe..788ad41 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
@@ -51,8 +51,9 @@ public class TestMapReduceAggregates extends TestCase {
     FileSystem fs = FileSystem.get(conf);
     int numOfInputLines = 20;
 
-    Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test");
-    Path INPUT_DIR = new Path("build/test/input_for_aggregates_test");
+    String baseDir = System.getProperty("test.build.data", "build/test/data");
+    Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test");
+    Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test");
     String inputFile = "input.txt";
     fs.delete(INPUT_DIR, true);
     fs.mkdirs(INPUT_DIR);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f22ce96b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
index b49f2d8..1fca5c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
@@ -1772,7 +1772,9 @@ public class TestCombineFileInputFormat {
     // default fs path
     assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString());
     // add a local file
-    Path localPath = new Path("testFile1");
+    String localPathRoot = System.getProperty("test.build.data",
+        "build/test/data");
+    Path localPath = new Path(localPathRoot, "testFile1");
     FileSystem lfs = FileSystem.getLocal(conf);
     FSDataOutputStream dos = lfs.create(localPath);
     dos.writeChars("Local file for CFIF");


Mime
View raw message