hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [04/50] [abbrv] MAPREDUCE-5996. native-task: Rename system tests into standard directory layout. Contributed by Todd Lipcon.
Date Sat, 13 Sep 2014 03:54:10 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
deleted file mode 100644
index b665971..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ResultVerifier.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import java.io.IOException;
-import java.util.zip.CRC32;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-
-public class ResultVerifier {
-  /**
-   * verify the result
-   * 
-   * @param sample
-   *          :nativetask output
-   * @param source
-   *          :yuanwenjian
-   * @throws Exception
-   */
-  public static boolean verify(String sample, String source) throws Exception {
-    FSDataInputStream sourcein = null;
-    FSDataInputStream samplein = null;
-
-    final Configuration conf = new Configuration();
-    final FileSystem fs = FileSystem.get(conf);
-    final Path hdfssource = new Path(source);
-    final Path[] sourcepaths = FileUtil.stat2Paths(fs.listStatus(hdfssource));
-
-    final Path hdfssample = new Path(sample);
-    final Path[] samplepaths = FileUtil.stat2Paths(fs.listStatus(hdfssample));
-    if (sourcepaths == null) {
-      throw new Exception("source file can not be found");
-    }
-    if (samplepaths == null) {
-      throw new Exception("sample file can not be found");
-    }
-    if (sourcepaths.length != samplepaths.length) {
-      return false;
-    }
-    for (int i = 0; i < sourcepaths.length; i++) {
-      final Path sourcepath = sourcepaths[i];
-      // op result file start with "part-r" like part-r-00000
-
-      if (!sourcepath.getName().startsWith("part-r")) {
-        continue;
-      }
-      Path samplepath = null;
-      for (int j = 0; j < samplepaths.length; j++) {
-        if (samplepaths[i].getName().equals(sourcepath.getName())) {
-          samplepath = samplepaths[i];
-          break;
-        }
-      }
-      if (samplepath == null) {
-        throw new Exception("cound not found file " + samplepaths[0].getParent() + "/" +
sourcepath.getName()
-            + " , as sourcepaths has such file");
-      }
-
-      // compare
-      try {
-        if (fs.exists(sourcepath) && fs.exists(samplepath)) {
-          sourcein = fs.open(sourcepath);
-          samplein = fs.open(samplepath);
-        } else {
-          System.err.println("result file not found:" + sourcepath + " or " + samplepath);
-          return false;
-        }
-
-        CRC32 sourcecrc, samplecrc;
-        samplecrc = new CRC32();
-        sourcecrc = new CRC32();
-        final byte[] bufin = new byte[1 << 16];
-        int readnum = 0;
-        int totalRead = 0;
-        while (samplein.available() > 0) {
-          readnum = samplein.read(bufin);
-          totalRead += readnum;
-          samplecrc.update(bufin, 0, readnum);
-        }
-
-        if (0 == totalRead) {
-          throw new Exception("source " + sample + " is empty file");
-        }
-
-        totalRead = 0;
-        while (sourcein.available() > 0) {
-          readnum = sourcein.read(bufin);
-          totalRead += readnum;
-          sourcecrc.update(bufin, 0, readnum);
-        }
-        if (0 == totalRead) {
-          throw new Exception("source " + sample + " is empty file");
-        }
-
-        if (samplecrc.getValue() == sourcecrc.getValue()) {
-          ;
-        } else {
-          return false;
-        }
-      } catch (final IOException e) {
-        throw new Exception("verify exception :", e);
-      } finally {
-
-        try {
-          if (samplein != null) {
-            samplein.close();
-          }
-          if (sourcein != null) {
-            sourcein.close();
-          }
-        } catch (final IOException e) {
-          e.printStackTrace();
-        }
-
-      }
-    }
-    return true;
-  }
-
-  public static void main(String[] args) {
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
deleted file mode 100644
index 9cc473d..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/ScenarioConfiguration.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-import org.apache.hadoop.conf.Configuration;
-
-public class ScenarioConfiguration extends Configuration {
-  public ScenarioConfiguration() {
-    super();
-    this.addResource(TestConstants.COMMON_CONF_PATH);
-  }
-
-  public void addcombinerConf() {
-    this.addResource(TestConstants.COMBINER_CONF_PATH);
-  }
-
-  public void addKVTestConf() {
-    this.addResource(TestConstants.KVTEST_CONF_PATH);
-  }
-  
-  public void addNonSortTestConf() {
-    this.addResource(TestConstants.NONSORT_TEST_CONF);
-  }
-
-  public void addNativeConf() {
-    this.set(TestConstants.NATIVETASK_COLLECTOR_DELEGATOR,
-        TestConstants.NATIVETASK_COLLECTOR_DELEGATOR_CLASS);
-  }
-
-  public static Configuration getNormalConfiguration() {
-    Configuration normalConf = new Configuration();
-    normalConf.addResource("common_conf.xml");
-    normalConf.addResource("normal_conf.xml");
-    return normalConf;
-  }
-
-  public static Configuration getNativeConfiguration() {
-    Configuration nativeConf = new Configuration();
-    nativeConf.addResource("common_conf.xml");
-    nativeConf.addResource("native_conf.xml");
-    return nativeConf;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2cba48f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
deleted file mode 100644
index 3beb082..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/system/function/org/apache/hadoop/mapred/nativetask/testutil/TestConstants.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.mapred.nativetask.testutil;
-
-public class TestConstants {
-  // conf path
-  public static final String COMBINER_CONF_PATH = "test-combiner-conf.xml";
-  public static final String KVTEST_CONF_PATH = "kvtest-conf.xml";
-  public static final String NONSORT_TEST_CONF = "test-nonsort-conf.xml";
-
-  public static final String NATIVETASK_KVSIZE_MIN = "nativetask.kvsize.min";
-  public static final String NATIVETASK_KVSIZE_MAX = "nativetask.kvsize.max";
-
-  public static final String NATIVETASK_KVTEST_INPUTDIR = "nativetask.kvtest.inputdir";
-  public static final String NATIVETASK_KVTEST_OUTPUTDIR = "nativetask.kvtest.outputdir";
-  public static final String NATIVETASK_KVTEST_NORMAL_OUTPUTDIR = "normal.kvtest.outputdir";
-  public static final String NATIVETASK_KVTEST_CREATEFILE = "nativetask.kvtest.createfile";
-  public static final String NATIVETASK_KVTEST_FILE_RECORDNUM = "nativetask.kvtest.file.recordnum";
-  public static final String NATIVETASK_KVTEST_KEYCLASSES = "nativetask.kvtest.keyclasses";
-  public static final String NATIVETASK_KVTEST_VALUECLASSES = "nativetask.kvtest.valueclasses";
-  public static final String NATIVETASK_COLLECTOR_DELEGATOR = "mapreduce.map.output.collector.delegator.class";
-  public static final String NATIVETASK_COLLECTOR_DELEGATOR_CLASS = "org.apache.hadoop.mapred.nativetask.testutil.EnforceNativeOutputCollectorDelegator";
-
-  public static final String SNAPPY_COMPRESS_CONF_PATH = "test-snappy-compress-conf.xml";
-  public static final String GZIP_COMPRESS_CONF_PATH = "test-gzip-compress-conf.xml";
-  public static final String BZIP2_COMPRESS_CONF_PATH = "test-bzip2-compress-conf.xml";
-  public static final String DEFAULT_COMPRESS_CONF_PATH = "test-default-compress-conf.xml";
-  public static final String LZ4_COMPRESS_CONF_PATH = "test-lz4-compress-conf.xml";
-  public static final String NATIVETASK_COMPRESS_FILESIZE = "nativetask.compress.filesize";
-
-  public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_KEY = "nativetask.combinertest.inputpath";
-  public static final String NATIVETASK_TEST_COMBINER_INPUTPATH_DEFAULTV = "./combinertest/input";
-  public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH = "nativetask.combinertest.outputdir";
-  public static final String NATIVETASK_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/native";
-  public static final String NORMAL_TEST_COMBINER_OUTPUTPATH = "normal.combinertest.outputdir";
-  public static final String NORMAL_TEST_COMBINER_OUTPUTPATH_DEFAULTV = "./combinertest/output/normal";
-  public static final String OLDAPI_NATIVETASK_TEST_COMBINER_OUTPUTPATH = "oldAPI.nativetask.combinertest.outputdir";
-  public static final String OLDAPI_NORMAL_TEST_COMBINER_OUTPUTPATH = "oldAPI.normal.combinertest.outputdir";
-  public static final String NATIVETASK_COMBINER_WORDCOUNT_FILESIZE = "nativetask.combiner.wordcount.filesize";
-  public static final String NATIVETASK_NONSORTTEST_FILESIZE = "nativetask.nonsorttest.filesize";
-
-  public static final String COMMON_CONF_PATH = "common_conf.xml";
-
-  public static final String FILESIZE_KEY = "kvtest.file.size";
-  public static final String NATIVETASK_KVSIZE_MAX_LARGEKV_TEST = "nativetask.kvsize.max.largekv";
-
-  public static final String NATIVETASK_MAP_OUTPUT_SORT = "mapreduce.sort.avoidance";
-  public static final String NONSORT_TEST_INPUTDIR = "nativetask.nonsorttest.inputpath";
-  public static final String NONSORT_TEST_NATIVE_OUTPUT = "nonsorttest.native.outputdir";
-  public static final String NONSORT_TEST_NORMAL_OUTPUT = "nonsorttest.normal.outputdir";
-
-}


Mime
View raw message