hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [05/50] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
Date Sat, 13 Sep 2014 03:54:11 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
deleted file mode 100644
index 8a4aa6f..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/CombinerTest.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.IntSumReducer;
-import org.apache.hadoop.mapred.nativetask.combinertest.WordCount.TokenizerMapper;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CombinerTest {
-  private FileSystem fs;
-  private String inputpath;
-  private String nativeoutputpath;
-  private String hadoopoutputpath;
-
-  @Test
-  public void testWordCountCombiner() {
-    try {
-
-      final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
-      nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
-      final Job nativejob = getJob("nativewordcount", nativeConf, inputpath, nativeoutputpath);
-
-      final Configuration commonConf = ScenarioConfiguration.getNormalConfiguration();
-      commonConf.addResource(TestConstants.COMBINER_CONF_PATH);
-
-      final Job normaljob = getJob("normalwordcount", commonConf, inputpath, hadoopoutputpath);
-
-      nativejob.waitForCompletion(true);
-            
-      Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-      
-      normaljob.waitForCompletion(true);
-      Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-       
-      assertEquals(true, ResultVerifier.verify(nativeoutputpath, hadoopoutputpath));
-      assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter", 
-          nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-      
-    } catch (final Exception e) {
-      e.printStackTrace();
-      assertEquals("run exception", true, false);
-    }
-  }
-
-  @Before
-  public void startUp() throws Exception {
-    final ScenarioConfiguration conf = new ScenarioConfiguration();
-    conf.addcombinerConf();
-
-    this.fs = FileSystem.get(conf);
-
-    this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
-        TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
-
-    if (!fs.exists(new Path(inputpath))) {
-      new TestInputFile(
-          conf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE, 1000000),
-          Text.class.getName(),
-          Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
-    }
-
-    this.nativeoutputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
-        TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativewordcount";
-    this.hadoopoutputpath = conf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
-        TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalwordcount";
-  }
-
-  protected static Job getJob(String jobname, Configuration inputConf, String inputpath, String outputpath)
-      throws Exception {
-    final Configuration conf = new Configuration(inputConf);
-    conf.set("fileoutputpath", outputpath);
-    final FileSystem fs = FileSystem.get(conf);
-    if (fs.exists(new Path(outputpath))) {
-      fs.delete(new Path(outputpath));
-    }
-    fs.close();
-    final Job job = new Job(conf, jobname);
-    job.setJarByClass(WordCount.class);
-    job.setMapperClass(TokenizerMapper.class);
-    job.setCombinerClass(IntSumReducer.class);
-    job.setReducerClass(IntSumReducer.class);
-    job.setOutputKeyClass(Text.class);
-    job.setOutputValueClass(IntWritable.class);
-    job.setInputFormatClass(SequenceFileInputFormat.class);
-    FileInputFormat.addInputPath(job, new Path(inputpath));
-    FileOutputFormat.setOutputPath(job, new Path(outputpath));
-    return job;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
deleted file mode 100644
index 50953e0..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Test;
-
-public class LargeKVCombinerTest {
-
-  @Test
-  public void testLargeValueCombiner(){
-    final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
-    final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
-    normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
-    nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
-    final int deafult_KVSize_Maximum = 1 << 22; // 4M
-    final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
-        deafult_KVSize_Maximum);
-    final String inputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
-        TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/largeKV";
-    final String nativeOutputPath = normalConf.get(TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH,
-        TestConstants.NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/nativeLargeKV";
-    final String hadoopOutputPath = normalConf.get(TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH,
-        TestConstants.NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV) + "/normalLargeKV";
-    try {
-      final FileSystem fs = FileSystem.get(normalConf);
-      for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
-        
-        int max = i;
-        int min = Math.max(i / 4, max - 10);
-        
-        System.out.println("===KV Size Test: min size: " + min + ", max size: " + max);
-        
-        normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
-        normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
-        nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
-        nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
-        fs.delete(new Path(inputPath), true);
-        new TestInputFile(normalConf.getInt(TestConstants.NATIVETASK_COMBINER_WORDCOUNT_FILESIZE,
-            1000000), IntWritable.class.getName(),
-            Text.class.getName(), normalConf).createSequenceTestFile(inputPath, 1);
-        
-        final Job normaljob = CombinerTest.getJob("normalwordcount", normalConf, inputPath, hadoopOutputPath);
-        final Job nativejob = CombinerTest.getJob("nativewordcount", nativeConf, inputPath, nativeOutputPath);
-        
-        nativejob.waitForCompletion(true);
-        Counter nativeReduceGroups = nativejob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-        
-        normaljob.waitForCompletion(true);
-        Counter normalReduceGroups = normaljob.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-        
-        final boolean compareRet = ResultVerifier.verify(nativeOutputPath, hadoopOutputPath);
-        
-        final String reason = "LargeKVCombinerTest failed with, min size: " + min
-            + ", max size: " + max + ", normal out: " + hadoopOutputPath + ", native Out: " + nativeOutputPath;
-        
-        assertEquals(reason, true, compareRet);
-//        assertEquals("Native Reduce reduce group counter should equal orignal reduce group counter", 
-//            nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-      }
-      fs.close();
-    } catch (final Exception e) {
-      e.printStackTrace();
-      assertEquals("run exception", true, false);
-    }
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
deleted file mode 100644
index 5691e02..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/OldAPICombinerTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Counter;
-import org.junit.Before;
-import org.junit.Test;
-
-public class OldAPICombinerTest {
-  private FileSystem fs;
-  private String inputpath;
-
-  @Test
-  public void testWordCountCombinerWithOldAPI() throws Exception {
-    final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
-    nativeConf.addResource(TestConstants.COMBINER_CONF_PATH);
-    final String nativeoutput = nativeConf.get(TestConstants.OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH);
-    final JobConf nativeJob = getOldAPIJobconf(nativeConf, "nativeCombinerWithOldAPI", inputpath, nativeoutput);
-    RunningJob nativeRunning = JobClient.runJob(nativeJob);
-
-    Counter nativeReduceGroups = nativeRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-    
-    final Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
-    normalConf.addResource(TestConstants.COMBINER_CONF_PATH);
-    final String normaloutput = normalConf.get(TestConstants.OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH);
-    final JobConf normalJob = getOldAPIJobconf(normalConf, "normalCombinerWithOldAPI", inputpath, normaloutput);
-    
-    RunningJob normalRunning = JobClient.runJob(normalJob);
-    Counter normalReduceGroups = normalRunning.getCounters().findCounter(Task.Counter.REDUCE_INPUT_RECORDS);
-    
-    final boolean compareRet = ResultVerifier.verify(nativeoutput, normaloutput);
-    assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-    
-    assertEquals("The input reduce record count must be same", nativeReduceGroups.getValue(), normalReduceGroups.getValue());
-  }
-
-  @Before
-  public void startUp() throws Exception {
-    final ScenarioConfiguration conf = new ScenarioConfiguration();
-    conf.addcombinerConf();
-    this.fs = FileSystem.get(conf);
-    this.inputpath = conf.get(TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_KEY,
-        TestConstants.NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV) + "/wordcount";
-
-    if (!fs.exists(new Path(inputpath))) {
-      new TestInputFile(conf.getInt("nativetask.combiner.wordcount.filesize", 1000000), Text.class.getName(),
-          Text.class.getName(), conf).createSequenceTestFile(inputpath, 1, (byte)('a'));
-    }
-  }
-
-  private static JobConf getOldAPIJobconf(Configuration configuration, String name, String input, String output)
-      throws Exception {
-    final JobConf jobConf = new JobConf(configuration);
-    final FileSystem fs = FileSystem.get(configuration);
-    if (fs.exists(new Path(output))) {
-      fs.delete(new Path(output), true);
-    }
-    fs.close();
-    jobConf.setJobName(name);
-    jobConf.setOutputKeyClass(Text.class);
-    jobConf.setOutputValueClass(IntWritable.class);
-    jobConf.setMapperClass(WordCountWithOldAPI.TokenizerMapperWithOldAPI.class);
-    jobConf.setCombinerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
-    jobConf.setReducerClass(WordCountWithOldAPI.IntSumReducerWithOldAPI.class);
-
-    jobConf.setInputFormat(SequenceFileInputFormat.class);
-    jobConf.setOutputFormat(TextOutputFormat.class);
-
-    FileInputFormat.setInputPaths(jobConf, new Path(input));
-    FileOutputFormat.setOutputPath(jobConf, new Path(output));
-    return jobConf;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
deleted file mode 100644
index 490b82b..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-
-public class WordCount {
-
-  private static Log LOG = LogFactory.getLog(WordCount.class);
-  
-  public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
-
-    private final static IntWritable one = new IntWritable(1);
-    private final Text word = new Text();
-
-    @Override
-    public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
-      final StringTokenizer itr = new StringTokenizer(value.toString());
-      while (itr.hasMoreTokens()) {
-        word.set(itr.nextToken());
-        context.write(word, one);
-      }
-    }
-  }
-
-  public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
-    private final IntWritable result = new IntWritable();
-
-    @Override
-    public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
-    InterruptedException {
-      int sum = 0;
-      for (final IntWritable val : values) {
-        sum += val.get();
-      }
-      result.set(sum);
-      context.write(key, result);
-    }
-  }
-
-  public static void main(String[] args) throws Exception {
-    final Configuration conf = new Configuration();
-    final String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
-    if (otherArgs.length != 2) {
-      System.err.println("Usage: wordcount <in> <out>");
-      System.exit(2);
-    }
-    final Job job = new Job(conf, conf.get(MRJobConfig.JOB_NAME, "word count"));
-    job.setJarByClass(WordCount.class);
-    job.setMapperClass(TokenizerMapper.class);
-    job.setCombinerClass(IntSumReducer.class);
-    job.setReducerClass(IntSumReducer.class);
-    job.setOutputKeyClass(Text.class);
-    job.setOutputValueClass(IntWritable.class);
-    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
-    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
-    System.exit(job.waitForCompletion(true) ? 0 : 1);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
deleted file mode 100644
index a11ea91..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/combinertest/WordCountWithOldAPI.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.combinertest;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.Reporter;
-
-public class WordCountWithOldAPI {
-
-  public static class TokenizerMapperWithOldAPI extends MapReduceBase implements
-  Mapper<Object, Text, Text, IntWritable> {
-    private final static IntWritable one = new IntWritable(1);
-    private final Text word = new Text();
-
-    @Override
-    public void map(Object key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter)
-        throws IOException {
-      final StringTokenizer itr = new StringTokenizer(value.toString());
-      while (itr.hasMoreTokens()) {
-        word.set(itr.nextToken());
-        output.collect(word, one);
-      }
-    }
-  }
-
-  public static class IntSumReducerWithOldAPI extends MapReduceBase implements
-  Reducer<Text, IntWritable, Text, IntWritable> {
-    private final IntWritable result = new IntWritable();
-
-    @Override
-    public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output,
-        Reporter reporter) throws IOException {
-      int sum = 0;
-      while (values.hasNext()) {
-        sum += values.next().get();
-      }
-      result.set(sum);
-      output.collect(key, result);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
deleted file mode 100644
index 83c4794..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressMapper.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.compresstest;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class CompressMapper {
-  public static final String inputFile = "./compress/input.txt";
-  public static final String outputFileDir = "./compress/output/";
-
-  public static class TextCompressMapper extends Mapper<Text, Text, Text, Text> {
-
-    @Override
-    protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
-       context.write(key, value);
-    }
-  }
-
-  public static Job getCompressJob(String jobname, Configuration conf) {
-    Job job = null;
-    try {
-      job = new Job(conf, jobname + "-CompressMapperJob");
-      job.setJarByClass(CompressMapper.class);
-      job.setMapperClass(TextCompressMapper.class);
-      job.setOutputKeyClass(Text.class);
-      job.setMapOutputValueClass(Text.class);
-      final Path outputpath = new Path(outputFileDir + jobname);
-      // if output file exists ,delete it
-      final FileSystem hdfs = FileSystem.get(new ScenarioConfiguration());
-      if (hdfs.exists(outputpath)) {
-        hdfs.delete(outputpath);
-      }
-      hdfs.close();
-      job.setInputFormatClass(SequenceFileInputFormat.class);
-      FileInputFormat.addInputPath(job, new Path(inputFile));
-      FileOutputFormat.setOutputPath(job, outputpath);
-    } catch (final Exception e) {
-      e.printStackTrace();
-    }
-    return job;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
deleted file mode 100644
index 0406375..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/compresstest/CompressTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.compresstest;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.junit.Before;
-import org.junit.Test;
-
-public class CompressTest {
-
-  @Test
-  public void testSnappyCompress() throws Exception {
-    final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
-    conf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
-    final Job job = CompressMapper.getCompressJob("nativesnappy", conf);
-    job.waitForCompletion(true);
-
-    final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
-    hadoopconf.addResource(TestConstants.SNAPPY_COMPRESS_CONF_PATH);
-    final Job hadoopjob = CompressMapper.getCompressJob("hadoopsnappy", hadoopconf);
-    hadoopjob.waitForCompletion(true);
-
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativesnappy",
-        CompressMapper.outputFileDir + "hadoopsnappy");
-    assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-  }
-
-  @Test
-  public void testGzipCompress() throws Exception {
-    final Configuration conf = ScenarioConfiguration.getNativeConfiguration();
-    conf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
-    final Job job = CompressMapper.getCompressJob("nativegzip", conf);
-    job.waitForCompletion(true);
-
-    final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
-    hadoopconf.addResource(TestConstants.GZIP_COMPRESS_CONF_PATH);
-    final Job hadoopjob = CompressMapper.getCompressJob("hadoopgzip", hadoopconf);
-    hadoopjob.waitForCompletion(true);
-
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativegzip",
-        CompressMapper.outputFileDir + "hadoopgzip");
-    assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-  }
-
-  @Test
-  public void testBzip2Compress() throws Exception {
-    final Configuration nativeconf = ScenarioConfiguration.getNativeConfiguration();
-    nativeconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
-    final Job nativejob = CompressMapper.getCompressJob("nativebzip2", nativeconf);
-    nativejob.waitForCompletion(true);
-
-    final Configuration hadoopconf = ScenarioConfiguration.getNormalConfiguration();
-    hadoopconf.addResource(TestConstants.BZIP2_COMPRESS_CONF_PATH);
-    final Job hadoopjob = CompressMapper.getCompressJob("hadoopbzip2", hadoopconf);
-    hadoopjob.waitForCompletion(true);
-
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativebzip2",
-        CompressMapper.outputFileDir + "hadoopbzip2");
-    assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-  }
-
-  @Test
-  public void testLz4Compress() throws Exception {
-    final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
-    nativeConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
-    final Job nativeJob = CompressMapper.getCompressJob("nativelz4", nativeConf);
-    nativeJob.waitForCompletion(true);
-
-    final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
-    hadoopConf.addResource(TestConstants.LZ4_COMPRESS_CONF_PATH);
-    final Job hadoopJob = CompressMapper.getCompressJob("hadooplz4", hadoopConf);
-    hadoopJob.waitForCompletion(true);
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativelz4",
-        CompressMapper.outputFileDir + "hadooplz4");
-    assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-  }
-
-  @Test
-  public void testDefaultCompress() throws Exception {
-    final Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
-    nativeConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
-    final Job nativeJob = CompressMapper.getCompressJob("nativedefault", nativeConf);
-    nativeJob.waitForCompletion(true);
-
-    final Configuration hadoopConf = ScenarioConfiguration.getNormalConfiguration();
-    hadoopConf.addResource(TestConstants.DEFAULT_COMPRESS_CONF_PATH);
-    final Job hadoopJob = CompressMapper.getCompressJob("hadoopdefault", hadoopConf);
-    hadoopJob.waitForCompletion(true);
-    final boolean compareRet = ResultVerifier.verify(CompressMapper.outputFileDir + "nativedefault",
-        CompressMapper.outputFileDir + "hadoopdefault");
-    assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-  }
-
-  @Before
-  public void startUp() throws Exception {
-    final ScenarioConfiguration conf = new ScenarioConfiguration();
-    final FileSystem fs = FileSystem.get(conf);
-    final Path path = new Path(CompressMapper.inputFile);
-    fs.delete(path);
-    if (!fs.exists(path)) {
-      new TestInputFile(ScenarioConfiguration.getNormalConfiguration().getInt(
-          TestConstants.NATIVETASK_COMPRESS_FILESIZE, 100000),
-          Text.class.getName(), Text.class.getName(), conf)
-      .createSequenceTestFile(CompressMapper.inputFile);
-
-    }
-    fs.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
deleted file mode 100644
index ce23780..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/HashSumReducer.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Reducer;
-
-public class HashSumReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, IntWritable> {
-
-  ByteArrayOutputStream os = new ByteArrayOutputStream();
-  DataOutputStream dos = new DataOutputStream(os);
-
-  @Override
-  public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
-    int hashSum = 0;
-    for (final VTYPE val : values) {
-      if (val instanceof Writable) {
-        os.reset();
-        ((Writable) val).write(dos);
-        final int hash = Arrays.hashCode(os.toByteArray());
-        hashSum += hash;
-      }
-    }
-
-    context.write(key, new IntWritable(hashSum));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
deleted file mode 100644
index 6d683f8..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.IOException;
-import java.util.zip.CRC32;
-
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class KVJob {
-  public static final String INPUTPATH = "nativetask.kvtest.inputfile.path";
-  public static final String OUTPUTPATH = "nativetask.kvtest.outputfile.path";
-  Job job = null;
-
-  public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> {
-    @Override
-    public void map(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
-      context.write(key, value);
-    }
-  }
-
-  public static class KVMReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
-    public void reduce(KTYPE key, VTYPE value, Context context) throws IOException, InterruptedException {
-      context.write(key, value);
-    }
-  }
-
-  public static class KVReducer<KTYPE, VTYPE> extends Reducer<KTYPE, VTYPE, KTYPE, VTYPE> {
-
-    @Override
-    public void reduce(KTYPE key, Iterable<VTYPE> values, Context context) throws IOException, InterruptedException {
-      long resultlong = 0;// 8 bytes match BytesFactory.fromBytes function
-      final CRC32 crc32 = new CRC32();
-      for (final VTYPE val : values) {
-        crc32.reset();
-        crc32.update(BytesFactory.toBytes(val));
-        resultlong += crc32.getValue();
-      }
-      final VTYPE V = null;
-      context.write(key, (VTYPE) BytesFactory.newObject(Longs.toByteArray(resultlong), V.getClass().getName()));
-    }
-  }
-
-  public KVJob(String jobname, Configuration conf, Class<?> keyclass, Class<?> valueclass, String inputpath,
-      String outputpath) throws Exception {
-    job = new Job(conf, jobname);
-    job.setJarByClass(KVJob.class);
-    job.setMapperClass(KVJob.ValueMapper.class);
-    job.setOutputKeyClass(keyclass);
-    job.setMapOutputValueClass(valueclass);
-    
-    if (conf.get(TestConstants.NATIVETASK_KVTEST_CREATEFILE).equals("true")) {
-      final FileSystem fs = FileSystem.get(conf);
-      fs.delete(new Path(inputpath), true);
-      fs.close();
-      final TestInputFile testfile = new TestInputFile(Integer.valueOf(conf.get(
-          TestConstants.FILESIZE_KEY, "1000")),
-          keyclass.getName(), valueclass.getName(), conf);
-      testfile.createSequenceTestFile(inputpath);
-
-    }
-    job.setInputFormatClass(SequenceFileInputFormat.class);
-    FileInputFormat.addInputPath(job, new Path(inputpath));
-    FileOutputFormat.setOutputPath(job, new Path(outputpath));
-  }
-
-  public void runJob() throws Exception {
-
-    job.waitForCompletion(true);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
deleted file mode 100644
index 1e08854..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class KVTest {
-  private static Class<?>[] keyclasses = null;
-  private static Class<?>[] valueclasses = null;
-  private static String[] keyclassNames = null;
-  private static String[] valueclassNames = null;
-
-  private static Configuration nativekvtestconf = ScenarioConfiguration.getNativeConfiguration();
-  private static Configuration hadoopkvtestconf = ScenarioConfiguration.getNormalConfiguration();
-  static {
-    nativekvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
-    hadoopkvtestconf.addResource(TestConstants.KVTEST_CONF_PATH);
-  }
-
-  @Parameters(name = "key:{0}\nvalue:{1}")
-  public static Iterable<Class<?>[]> data() {
-    final String valueclassesStr = nativekvtestconf
-        .get(TestConstants.NATIVETASK_KVTEST_VALUECLASSES);
-    System.out.println(valueclassesStr);
-    valueclassNames = valueclassesStr.replaceAll("\\s", "").split(";");// delete
-    // " "
-    final ArrayList<Class<?>> tmpvalueclasses = new ArrayList<Class<?>>();
-    for (int i = 0; i < valueclassNames.length; i++) {
-      try {
-        if (valueclassNames[i].equals("")) {
-          continue;
-        }
-        tmpvalueclasses.add(Class.forName(valueclassNames[i]));
-      } catch (final ClassNotFoundException e) {
-        e.printStackTrace();
-      }
-    }
-    valueclasses = tmpvalueclasses.toArray(new Class[tmpvalueclasses.size()]);
-    final String keyclassesStr = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_KEYCLASSES);
-    System.out.println(keyclassesStr);
-    keyclassNames = keyclassesStr.replaceAll("\\s", "").split(";");// delete
-    // " "
-    final ArrayList<Class<?>> tmpkeyclasses = new ArrayList<Class<?>>();
-    for (int i = 0; i < keyclassNames.length; i++) {
-      try {
-        if (keyclassNames[i].equals("")) {
-          continue;
-        }
-        tmpkeyclasses.add(Class.forName(keyclassNames[i]));
-      } catch (final ClassNotFoundException e) {
-        e.printStackTrace();
-      }
-    }
-    keyclasses = tmpkeyclasses.toArray(new Class[tmpkeyclasses.size()]);
-    final Class<?>[][] kvgroup = new Class<?>[keyclassNames.length * valueclassNames.length][2];
-    for (int i = 0; i < keyclassNames.length; i++) {
-      final int tmpindex = i * valueclassNames.length;
-      for (int j = 0; j < valueclassNames.length; j++) {
-        kvgroup[tmpindex + j][0] = keyclasses[i];
-        kvgroup[tmpindex + j][1] = valueclasses[j];
-      }
-    }
-    return Arrays.asList(kvgroup);
-  }
-
-  private final Class<?> keyclass;
-  private final Class<?> valueclass;
-
-  public KVTest(Class<?> keyclass, Class<?> valueclass) {
-    this.keyclass = keyclass;
-    this.valueclass = valueclass;
-
-  }
-
-  @Test
-  public void testKVCompability() {
-
-    try {
-      final String nativeoutput = this.runNativeTest(
-          "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
-      final String normaloutput = this.runNormalTest(
-          "Test:" + keyclass.getSimpleName() + "--" + valueclass.getSimpleName(), keyclass, valueclass);
-      final boolean compareRet = ResultVerifier.verify(normaloutput, nativeoutput);
-      final String input = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
-          + keyclass.getName()
-          + "/" + valueclass.getName();
-      if(compareRet){
-        final FileSystem fs = FileSystem.get(hadoopkvtestconf);
-        fs.delete(new Path(nativeoutput), true);
-        fs.delete(new Path(normaloutput), true);
-        fs.delete(new Path(input), true);
-        fs.close();
-      }
-      assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-    } catch (final IOException e) {
-      assertEquals("test run exception:", null, e);
-    } catch (final Exception e) {
-      assertEquals("test run exception:", null, e);
-    }
-  }
-
-  @Before
-  public void startUp() {
-
-  }
-
-  private String runNativeTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
-    final String inputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
-        + keyclass.getName()
-        + "/" + valueclass.getName();
-    final String outputpath = nativekvtestconf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/"
-        + keyclass.getName() + "/" + valueclass.getName();
-    // if output file exists ,then delete it
-    final FileSystem fs = FileSystem.get(nativekvtestconf);
-    fs.delete(new Path(outputpath));
-    fs.close();
-    nativekvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
-    try {
-      final KVJob keyJob = new KVJob(jobname, nativekvtestconf, keyclass, valueclass, inputpath, outputpath);
-      keyJob.runJob();
-    } catch (final Exception e) {
-      return "native testcase run time error.";
-    }
-    return outputpath;
-  }
-
-  private String runNormalTest(String jobname, Class<?> keyclass, Class<?> valueclass) throws IOException {
-    final String inputpath = hadoopkvtestconf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/"
-        + keyclass.getName()
-        + "/" + valueclass.getName();
-    final String outputpath = hadoopkvtestconf
-        .get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR)
-        + "/"
-        + keyclass.getName() + "/" + valueclass.getName();
-    // if output file exists ,then delete it
-    final FileSystem fs = FileSystem.get(hadoopkvtestconf);
-    fs.delete(new Path(outputpath));
-    fs.close();
-    hadoopkvtestconf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
-    try {
-      final KVJob keyJob = new KVJob(jobname, hadoopkvtestconf, keyclass, valueclass, inputpath, outputpath);
-      keyJob.runJob();
-    } catch (final Exception e) {
-      return "normal testcase run time error.";
-    }
-    return outputpath;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
deleted file mode 100644
index 900b058..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.junit.Test;
-
-public class LargeKVTest {
-
-  @Test
-  public void testKeySize() {
-    runKVSizeTests(Text.class, IntWritable.class);
-  }
-
-  @Test
-  public void testValueSize() {
-    runKVSizeTests(IntWritable.class, Text.class);
-  }
-
-  private static Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
-  private static Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
-  static {
-    nativeConf.addResource(TestConstants.KVTEST_CONF_PATH);
-    nativeConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "true");
-    normalConf.addResource(TestConstants.KVTEST_CONF_PATH);
-    normalConf.set(TestConstants.NATIVETASK_KVTEST_CREATEFILE, "false");
-  }
-
-  public void runKVSizeTests(Class<?> keyClass, Class<?> valueClass) {
-    if (!keyClass.equals(Text.class) && !valueClass.equals(Text.class)) {
-      return;
-    }
-    final int deafult_KVSize_Maximum = 1 << 22; // 4M
-    final int KVSize_Maximu = normalConf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX_LARGEKV_TEST,
-        deafult_KVSize_Maximum);
-    try {
-
-      for (int i = 65536; i <= KVSize_Maximu; i *= 4) {
-        int min = i / 4;
-        int max = i;
-        nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
-        nativeConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
-        normalConf.set(TestConstants.NATIVETASK_KVSIZE_MIN, String.valueOf(min));
-        normalConf.set(TestConstants.NATIVETASK_KVSIZE_MAX, String.valueOf(max));
-
-        System.out.println("===KV Size Test: min size: " + min + ", max size: " + max + ", keyClass: "
-            + keyClass.getName() + ", valueClass: " + valueClass.getName());
-
-        final String nativeOutPut = runNativeLargeKVTest("Test Large Value Size:" + String.valueOf(i), keyClass,
-            valueClass, nativeConf);
-        final String normalOutPut = this.runNormalLargeKVTest("Test Large Key Size:" + String.valueOf(i), keyClass,
-            valueClass, normalConf);
-        final boolean compareRet = ResultVerifier.verify(normalOutPut, nativeOutPut);
-        final String reason = "keytype: " + keyClass.getName() + ", valuetype: " + valueClass.getName()
-            + ", failed with " + (keyClass.equals(Text.class) ? "key" : "value") + ", min size: " + min
-            + ", max size: " + max + ", normal out: " + normalOutPut + ", native Out: " + nativeOutPut;
-        assertEquals(reason, true, compareRet);
-      }
-    } catch (final Exception e) {
-      // TODO: handle exception
-      // assertEquals("test run exception:", null, e);
-      e.printStackTrace();
-    }
-  }
-
-  private String runNativeLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
-      throws Exception {
-    final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
-        + "/" + valueclass.getName();
-    final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_OUTPUTDIR) + "/LargeKV/" + keyclass.getName()
-        + "/" + valueclass.getName();
-    // if output file exists ,then delete it
-    final FileSystem fs = FileSystem.get(conf);
-    fs.delete(new Path(outputpath), true);
-    fs.close();
-    try {
-      final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
-      keyJob.runJob();
-    } catch (final Exception e) {
-      return "normal testcase run time error.";
-    }
-    return outputpath;
-  }
-
-  private String runNormalLargeKVTest(String jobname, Class<?> keyclass, Class<?> valueclass, Configuration conf)
-      throws IOException {
-    final String inputpath = conf.get(TestConstants.NATIVETASK_KVTEST_INPUTDIR) + "/LargeKV/" + keyclass.getName()
-        + "/" + valueclass.getName();
-    final String outputpath = conf.get(TestConstants.NATIVETASK_KVTEST_NORMAL_OUTPUTDIR) + "/LargeKV/"
-        + keyclass.getName() + "/" + valueclass.getName();
-    // if output file exists ,then delete it
-    final FileSystem fs = FileSystem.get(conf);
-    fs.delete(new Path(outputpath), true);
-    fs.close();
-    try {
-      final KVJob keyJob = new KVJob(jobname, conf, keyclass, valueclass, inputpath, outputpath);
-      keyJob.runJob();
-    } catch (final Exception e) {
-      return "normal testcase run time error.";
-    }
-    return outputpath;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
deleted file mode 100644
index a194697..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.kvtest;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.VIntWritable;
-import org.apache.hadoop.io.VLongWritable;
-import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-
-
-public class TestInputFile {
-	
-  public static class KVSizeScope {
-    private static final int DefaultMinNum = 1;
-    private static final int DefaultMaxNum = 64;
-
-    public int minBytesNum;
-    public int maxBytesNum;
-
-    public KVSizeScope() {
-      this.minBytesNum = DefaultMinNum;
-      this.maxBytesNum = DefaultMaxNum;
-    }
-
-    public KVSizeScope(int min, int max) {
-      this.minBytesNum = min;
-      this.maxBytesNum = max;
-    }
-  }
-
-  private static HashMap<String, KVSizeScope> map = new HashMap<String, KVSizeScope>();
-
-  private byte[] databuf = null;
-  private final String keyClsName, valueClsName;
-  private int filesize = 0;
-  private int keyMaxBytesNum, keyMinBytesNum;
-  private int valueMaxBytesNum, valueMinBytesNum;
-  private SequenceFile.Writer writer = null;
-  Random r = new Random();
-  public static final int DATABUFSIZE = 1 << 22; // 4M
-
-  private enum State {
-    KEY, VALUE
-  };
-  
-  static {
-    map.put(BooleanWritable.class.getName(), new KVSizeScope(1, 1));
-    map.put(DoubleWritable.class.getName(), new KVSizeScope(8, 8));
-    map.put(FloatWritable.class.getName(), new KVSizeScope(4, 4));
-    map.put(VLongWritable.class.getName(), new KVSizeScope(8, 8));
-    map.put(ByteWritable.class.getName(), new KVSizeScope(1, 1));
-    map.put(LongWritable.class.getName(), new KVSizeScope(8, 8));
-    map.put(VIntWritable.class.getName(), new KVSizeScope(4, 4));
-    map.put(IntWritable.class.getName(), new KVSizeScope(4, 4));
-  }
-  
-  public TestInputFile(int filesize, String keytype, String valuetype, Configuration conf) throws Exception {
-    this.filesize = filesize;
-    this.databuf = new byte[DATABUFSIZE];
-    this.keyClsName = keytype;
-    this.valueClsName = valuetype;
-    final int defaultMinBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MIN, 1);
-    final int defaultMaxBytes = conf.getInt(TestConstants.NATIVETASK_KVSIZE_MAX, 64);
-
-    if (map.get(keytype) != null) {
-      keyMinBytesNum = map.get(keytype).minBytesNum;
-      keyMaxBytesNum = map.get(keytype).maxBytesNum;
-    } else {
-      keyMinBytesNum = defaultMinBytes;
-      keyMaxBytesNum = defaultMaxBytes;
-    }
-
-    if (map.get(valuetype) != null) {
-      valueMinBytesNum = map.get(valuetype).minBytesNum;
-      valueMaxBytesNum = map.get(valuetype).maxBytesNum;
-    } else {
-      valueMinBytesNum = defaultMinBytes;
-      valueMaxBytesNum = defaultMaxBytes;
-    }
-  }
-
-  public void createSequenceTestFile(String filepath) throws Exception {
-    int FULL_BYTE_SPACE = 256;
-    createSequenceTestFile(filepath, FULL_BYTE_SPACE);
-  }
-
-  public void createSequenceTestFile(String filepath, int base) throws Exception {
-    createSequenceTestFile(filepath, base, (byte)0);
-  }
-  
-  public void createSequenceTestFile(String filepath, int base,  byte start) throws Exception {
-    System.out.println("create file " + filepath);
-    System.out.println(keyClsName + " " + valueClsName);
-    Class<?> tmpkeycls, tmpvaluecls;
-    try {
-      tmpkeycls = Class.forName(keyClsName);
-    } catch (final ClassNotFoundException e) {
-      throw new Exception("key class not found: ", e);
-    }
-    try {
-      tmpvaluecls = Class.forName(valueClsName);
-    } catch (final ClassNotFoundException e) {
-      throw new Exception("key class not found: ", e);
-    }
-    try {
-      final Path outputfilepath = new Path(filepath);
-      final ScenarioConfiguration conf= new ScenarioConfiguration();
-      final FileSystem hdfs = outputfilepath.getFileSystem(conf);
-      writer = new SequenceFile.Writer(hdfs, conf, outputfilepath, tmpkeycls, tmpvaluecls);
-    } catch (final Exception e) {
-      e.printStackTrace();
-    }
-
-    int tmpfilesize = this.filesize;
-    while (tmpfilesize > DATABUFSIZE) {
-      nextRandomBytes(databuf, base, start);
-      final int size = flushBuf(DATABUFSIZE);
-      tmpfilesize -= size;
-    }
-    nextRandomBytes(databuf, base, start);
-    flushBuf(tmpfilesize);
-
-    if (writer != null) {
-      IOUtils.closeStream(writer);
-    } else {
-      throw new Exception("no writer to create sequenceTestFile!");
-    }
-  }
-  
-  private void nextRandomBytes(byte[] buf, int base) {
-    nextRandomBytes(buf, base, (byte)0);
-  }
-  
-  private void nextRandomBytes(byte[] buf, int base, byte start) {
-    r.nextBytes(buf);
-    for (int i = 0; i < buf.length; i++) {
-      buf[i] = (byte) ((buf[i] & 0xFF) % base + start);
-    }
-  }
-
-  private int flushBuf(int buflen) throws Exception {
-    final Random r = new Random();
-    int keybytesnum = 0;
-    int valuebytesnum = 0;
-    int offset = 0;
-
-    while (offset < buflen) {
-      final int remains = buflen - offset;
-      keybytesnum = keyMaxBytesNum;
-      if (keyMaxBytesNum != keyMinBytesNum) {
-        keybytesnum = keyMinBytesNum + r.nextInt(keyMaxBytesNum - keyMinBytesNum);
-      }
-
-      valuebytesnum = valueMaxBytesNum;
-      if (valueMaxBytesNum != valueMinBytesNum) {
-        valuebytesnum = valueMinBytesNum + r.nextInt(valueMaxBytesNum - valueMinBytesNum);
-      }
-
-      if (keybytesnum + valuebytesnum > remains) {
-        break;
-      }
-
-      final byte[] key = new byte[keybytesnum];
-      final byte[] value = new byte[valuebytesnum];
-
-      System.arraycopy(databuf, offset, key, 0, keybytesnum);
-      offset += keybytesnum;
-
-      System.arraycopy(databuf, offset, value, 0, valuebytesnum);
-      offset += valuebytesnum;
-      
-      try {
-        writer.append(BytesFactory.newObject(key, this.keyClsName), BytesFactory.newObject(value, this.valueClsName));
-      } catch (final IOException e) {
-        e.printStackTrace();
-        throw new Exception("sequence file create failed", e);
-      }
-    }
-    return offset;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
deleted file mode 100644
index b7b03e7..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.nonsorttest;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.nativetask.kvtest.TestInputFile;
-import org.apache.hadoop.mapred.nativetask.testutil.ResultVerifier;
-import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
-import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.junit.Before;
-import org.junit.Test;
-
-public class NonSortTest {
-
-  @Test
-  public void nonSortTest() throws Exception {
-    Configuration nativeConf = ScenarioConfiguration.getNativeConfiguration();
-    nativeConf.addResource(TestConstants.NONSORT_TEST_CONF);
-    nativeConf.set(TestConstants.NATIVETASK_MAP_OUTPUT_SORT, "false");
-    String inputpath = nativeConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
-    String outputpath = nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT);
-    final Job nativeNonSort = getJob(nativeConf, "NativeNonSort", inputpath, outputpath);
-    nativeNonSort.waitForCompletion(true);
-
-    Configuration normalConf = ScenarioConfiguration.getNormalConfiguration();
-    normalConf.addResource(TestConstants.NONSORT_TEST_CONF);
-    inputpath = normalConf.get(TestConstants.NONSORT_TEST_INPUTDIR);
-    outputpath = normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT);
-    final Job hadoopWithSort = getJob(normalConf, "NormalJob", inputpath, outputpath);
-    hadoopWithSort.waitForCompletion(true);
-
-    final boolean compareRet = ResultVerifier.verify(nativeConf.get(TestConstants.NONSORT_TEST_NATIVE_OUTPUT),
-        normalConf.get(TestConstants.NONSORT_TEST_NORMAL_OUTPUT));
-    assertEquals("file compare result: if they are the same ,then return true", true, compareRet);
-  }
-
-  @Before
-  public void startUp() throws Exception {
-    final ScenarioConfiguration configuration = new ScenarioConfiguration();
-    configuration.addNonSortTestConf();
-    final FileSystem fs = FileSystem.get(configuration);
-    final Path path = new Path(configuration.get(TestConstants.NONSORT_TEST_INPUTDIR));
-    if (!fs.exists(path)) {
-      new TestInputFile(configuration.getInt("nativetask.nonsorttest.filesize", 10000000), Text.class.getName(),
-          Text.class.getName(), configuration).createSequenceTestFile(path.toString());
-    }
-    fs.close();
-  }
-
-  private Job getJob(Configuration conf, String jobName, String inputpath, String outputpath) throws IOException {
-    final FileSystem fs = FileSystem.get(conf);
-    if (fs.exists(new Path(outputpath))) {
-      fs.delete(new Path(outputpath), true);
-    }
-    fs.close();
-    final Job job = new Job(conf, jobName);
-    job.setJarByClass(NonSortTestMR.class);
-    job.setMapperClass(NonSortTestMR.Map.class);
-    job.setReducerClass(NonSortTestMR.KeyHashSumReduce.class);
-    job.setOutputKeyClass(Text.class);
-    job.setMapOutputValueClass(IntWritable.class);
-    job.setOutputValueClass(LongWritable.class);
-    job.setInputFormatClass(SequenceFileInputFormat.class);
-    job.setOutputFormatClass(TextOutputFormat.class);
-    FileInputFormat.addInputPath(job, new Path(inputpath));
-    FileOutputFormat.setOutputPath(job, new Path(outputpath));
-    return job;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
deleted file mode 100644
index 4ca2449..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/nonsorttest/NonSortTestMR.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.nonsorttest;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-
-public class NonSortTestMR {
-
-  public static class Map extends Mapper<Object, Text, Text, IntWritable> {
-    private final static IntWritable one = new IntWritable(1);
-    private final Text word = new Text();
-
-    @Override
-    public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
-      final String line = value.toString();
-      final StringTokenizer tokenizer = new StringTokenizer(line);
-      while (tokenizer.hasMoreTokens()) {
-        word.set(tokenizer.nextToken());
-        context.write(word, one);
-      }
-    }
-  }
-
-  public static class KeyHashSumReduce extends Reducer<Text, IntWritable, Text, LongWritable> {
-    long sum = 0;
-    ByteArrayOutputStream os = new ByteArrayOutputStream();
-    DataOutputStream dos = new DataOutputStream(os);
-
-    @Override
-    public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException,
-    InterruptedException {
-      for (final IntWritable val : values) {
-        os.reset();
-        key.write(dos);
-        final int hash = Arrays.hashCode(os.toByteArray());
-        sum += hash;
-      }
-    }
-
-    @Override
-    public void cleanup(Context context) throws IOException, InterruptedException {
-      context.write(new Text("NonSortTest"), new LongWritable(sum));
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
deleted file mode 100644
index 5185371..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/BytesFactory.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.util.Random;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.ByteWritable;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.UTF8;
-import org.apache.hadoop.io.VIntWritable;
-import org.apache.hadoop.io.VLongWritable;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-
-
-public class BytesFactory {
-  public static Random r = new Random();
-
-  public static Object newObject(byte[] seed, String className) {
-    r.setSeed(seed.hashCode());
-    if (className.equals(IntWritable.class.getName())) {
-      return new IntWritable(Ints.fromByteArray(seed));
-    } else if (className.equals(FloatWritable.class.getName())) {
-      return new FloatWritable(r.nextFloat());
-    } else if (className.equals(DoubleWritable.class.getName())) {
-      return new DoubleWritable(r.nextDouble());
-    } else if (className.equals(LongWritable.class.getName())) {
-      return new LongWritable(Longs.fromByteArray(seed));
-    } else if (className.equals(VIntWritable.class.getName())) {
-      return new VIntWritable(Ints.fromByteArray(seed));
-    } else if (className.equals(VLongWritable.class.getName())) {
-      return new VLongWritable(Longs.fromByteArray(seed));
-    } else if (className.equals(BooleanWritable.class.getName())) {
-      return new BooleanWritable(seed[0] % 2 == 1 ? true : false);
-    } else if (className.equals(Text.class.getName())) {
-      return new Text(BytesUtil.toStringBinary(seed));
-    } else if (className.equals(ByteWritable.class.getName())) {
-      return new ByteWritable(seed.length > 0 ? seed[0] : 0);
-    } else if (className.equals(BytesWritable.class.getName())) {
-      return new BytesWritable(seed);
-    } else if (className.equals(UTF8.class.getName())) {
-      return new UTF8(BytesUtil.toStringBinary(seed));
-    } else if (className.equals(MockValueClass.class.getName())) {
-      return new MockValueClass(seed);
-    } else {
-      return null;
-    }
-  }
-
-
-  public static <VTYPE> byte[] fromBytes(byte[] bytes) throws Exception {
-    throw new Exception("Not supported");
-  }
-
-  public static <VTYPE> byte[] toBytes(VTYPE obj) {
-    final String className = obj.getClass().getName();
-    if (className.equals(IntWritable.class.getName())) {
-      return Ints.toByteArray(((IntWritable) obj).get());
-    } else if (className.equals(FloatWritable.class.getName())) {
-      return BytesUtil.toBytes(((FloatWritable) obj).get());
-    } else if (className.equals(DoubleWritable.class.getName())) {
-      return BytesUtil.toBytes(((DoubleWritable) obj).get());
-    } else if (className.equals(LongWritable.class.getName())) {
-      return Longs.toByteArray(((LongWritable) obj).get());
-    } else if (className.equals(VIntWritable.class.getName())) {
-      return Ints.toByteArray(((VIntWritable) obj).get());
-    } else if (className.equals(VLongWritable.class.getName())) {
-      return Longs.toByteArray(((VLongWritable) obj).get());
-    } else if (className.equals(BooleanWritable.class.getName())) {
-      return BytesUtil.toBytes(((BooleanWritable) obj).get());
-    } else if (className.equals(Text.class.getName())) {
-      return ((Text)obj).copyBytes();
-    } else if (className.equals(ByteWritable.class.getName())) {
-      return Ints.toByteArray((int) ((ByteWritable) obj).get());
-    } else if (className.equals(BytesWritable.class.getName())) {
-      // TODO: copyBytes instead?
-      return ((BytesWritable) obj).getBytes();
-    } else {
-      return new byte[0];
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
deleted file mode 100644
index a0f7d64..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.IOException;
-
-import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
-
-public class EnforceNativeOutputCollectorDelegator<K, V> extends NativeMapOutputCollectorDelegator<K, V> {
-  private boolean nativetaskloaded = false;
-
-  @Override
-  public void init(Context context)
- throws IOException, ClassNotFoundException {
-    try {
-      super.init(context);
-      nativetaskloaded = true;
-    } catch (final Exception e) {
-      nativetaskloaded = false;
-      System.err.println("load nativetask lib failed, Native-Task Delegation is disabled");
-      e.printStackTrace();
-    }
-  }
-
-  @Override
-  public void collect(K key, V value, int partition) throws IOException, InterruptedException {
-    if (this.nativetaskloaded) {
-      super.collect(key, value, partition);
-    } else {
-      // nothing to do.
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
deleted file mode 100644
index b27f00f..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/MockValueClass.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.nativetask.util.BytesUtil;
-
-public class MockValueClass implements Writable {
-  private final static int DEFAULT_ARRAY_LENGTH = 16;
-  private int a = 0;
-  private byte[] array;
-  private final LongWritable longWritable;
-  private final Text txt;
-  private final Random rand = new Random();
-
-  public MockValueClass() {
-    a = rand.nextInt();
-    array = new byte[DEFAULT_ARRAY_LENGTH];
-    rand.nextBytes(array);
-    longWritable = new LongWritable(rand.nextLong());
-    txt = new Text(BytesUtil.toStringBinary(array));
-  }
-
-  public MockValueClass(byte[] seed) {
-    a = seed.length;
-    array = new byte[seed.length];
-    System.arraycopy(seed, 0, array, 0, seed.length);
-    longWritable = new LongWritable(a);
-    txt = new Text(BytesUtil.toStringBinary(array));
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(a);
-    out.writeInt(array.length);
-    out.write(array);
-    longWritable.write(out);
-    txt.write(out);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    a = in.readInt();
-    final int length = in.readInt();
-    array = new byte[length];
-    in.readFully(array);
-    longWritable.readFields(in);
-    txt.readFields(in);
-  }
-}


Mime
View raw message