hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject [16/17] hadoop git commit: MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter)
Date Tue, 05 May 2015 23:20:56 GMT
MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml
(rchiang via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9809a16d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9809a16d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9809a16d

Branch: refs/heads/HDFS-7240
Commit: 9809a16d3c8068beccbf0106e99c7ede6ba11e0f
Parents: 0100b15
Author: Robert Kanter <rkanter@apache.org>
Authored: Mon May 4 17:48:10 2015 -0700
Committer: Robert Kanter <rkanter@apache.org>
Committed: Tue May 5 14:43:28 2015 -0700

----------------------------------------------------------------------
 .../conf/TestConfigurationFieldsBase.java       | 58 ++++++++++++++-
 hadoop-mapreduce-project/CHANGES.txt            |  3 +
 .../mapred/TestMapreduceConfigFields.java       | 76 ++++++++++++++++++++
 3 files changed, 135 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index c3fe3a3..2e4d8b1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -147,6 +147,12 @@ public abstract class TestConfigurationFieldsBase {
   private Set<String> xmlFieldsMissingInConfiguration = null;
 
   /**
+   * Member variable for debugging base class operation
+   */
+  protected boolean configDebug = false;
+  protected boolean xmlDebug = false;
+
+  /**
    * Abstract method to be used by subclasses for initializing base
    * members.
    */
@@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase {
     HashMap<String,String> retVal = new HashMap<String,String>();
 
     // Setup regexp for valid properties
-    String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$";
+    String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
     Pattern p = Pattern.compile(propRegex);
 
     // Iterate through class member variables
     int totalFields = 0;
     String value;
     for (Field f : fields) {
+      if (configDebug) {
+        System.out.println("Field: " + f);
+      }
       // Filter out anything that isn't "public static final"
       if (!Modifier.isStatic(f.getModifiers()) ||
           !Modifier.isPublic(f.getModifiers()) ||
@@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase {
       } catch (IllegalAccessException iaException) {
         continue;
       }
+      if (configDebug) {
+        System.out.println("  Value: " + value);
+      }
       // Special Case: Detect and ignore partial properties (ending in x)
       //               or file properties (ending in .xml)
       if (value.endsWith(".xml") ||
@@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase {
       //                  something like: blah.blah2(.blah3.blah4...)
       Matcher m = p.matcher(value);
       if (!m.find()) {
+        if (configDebug) {
+          System.out.println("  Passes Regex: false");
+        }
         continue;
       }
+      if (configDebug) {
+        System.out.println("  Passes Regex: true");
+      }
 
       // Save member variable/value as hash
-      retVal.put(value,f.getName());
+      if (!retVal.containsKey(value)) {
+        retVal.put(value,f.getName());
+      } else {
+        if (configDebug) {
+          System.out.println("ERROR: Already found key for property " + value);
+        }
+      }
     }
 
     return retVal;
@@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase {
       // Ignore known xml props
       if (xmlPropsToSkipCompare != null) {
         if (xmlPropsToSkipCompare.contains(key)) {
+          if (xmlDebug) {
+            System.out.println("  Skipping Full Key: " + key);
+          }
           continue;
         }
       }
@@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase {
 	}
       }
       if (skipPrefix) {
+        if (xmlDebug) {
+          System.out.println("  Skipping Prefix Key: " + key);
+        }
         continue;
       }
       if (conf.onlyKeyExists(key)) {
         retVal.put(key,null);
+        if (xmlDebug) {
+          System.out.println("  XML Key,Null Value: " + key);
+        }
       } else {
         String value = conf.get(key);
         if (value!=null) {
           retVal.put(key,entry.getValue());
+          if (xmlDebug) {
+            System.out.println("  XML Key,Valid Value: " + key);
+          }
         }
       }
       kvItr.remove();
@@ -312,6 +348,10 @@ public abstract class TestConfigurationFieldsBase {
 
     // Create class member/value map
     configurationMemberVariables = new HashMap<String,String>();
+    if (configDebug) {
+      System.out.println("Reading configuration classes");
+      System.out.println("");
+    }
     for (Class c : configurationClasses) {
       Field[] fields = c.getDeclaredFields();
       Map<String,String> memberMap =
@@ -320,9 +360,23 @@ public abstract class TestConfigurationFieldsBase {
         configurationMemberVariables.putAll(memberMap);
       }
     }
+    if (configDebug) {
+      System.out.println("");
+      System.out.println("=====");
+      System.out.println("");
+    }
 
     // Create XML key/value map
+    if (xmlDebug) {
+      System.out.println("Reading XML property files");
+      System.out.println("");
+    }
     xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
+    if (xmlDebug) {
+      System.out.println("");
+      System.out.println("=====");
+      System.out.println("");
+    }
 
     // Find class members not in the XML file
     configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 002fbe6..99621cb 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -304,6 +304,9 @@ Release 2.8.0 - UNRELEASED
     mapreduce.tasktracker.taskmemorymanager.monitoringinterval.
     (J.Andreina via aajisaka)
 
+    MAPREDUCE-6192. Create unit test to automatically compare
+    MR related classes and mapred-default.xml (rchiang via rkanter)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java
new file mode 100644
index 0000000..7f18714
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import java.util.HashSet;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+
+/**
+ * Unit test class to compare the following MR Configuration classes:
+ * <p></p>
+ * {@link org.apache.hadoop.mapreduce.MRJobConfig}
+ * {@link org.apache.hadoop.mapreduce.MRConfig}
+ * {@link org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig}
+ * {@link org.apache.hadoop.mapred.ShuffleHandler}
+ * {@link org.apache.hadoop.mapreduce.lib.output.FileOutputFormat}
+ * {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}
+ * {@link org.apache.hadoop.mapreduce.Job}
+ * {@link org.apache.hadoop.mapreduce.lib.input.NLineInputFormat}
+ * {@link org.apache.hadoop.mapred.JobConf}
+ * <p></p>
+ * against mapred-default.xml for missing properties.  Currently only
+ * throws an error if the class is missing a property.
+ * <p></p>
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestMapreduceConfigFields extends TestConfigurationFieldsBase {
+
+  @SuppressWarnings("deprecation")
+  @Override
+  public void initializeMemberVariables() {
+    xmlFilename = new String("mapred-default.xml");
+    configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class,
+        JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class,
+	FileInputFormat.class, Job.class, NLineInputFormat.class,
+	JobConf.class, FileOutputCommitter.class };
+
+    // Initialize used variables
+    configurationPropsToSkipCompare = new HashSet<String>();
+
+    // Set error modes
+    errorIfMissingConfigProps = true;
+    errorIfMissingXmlProps = false;
+
+    // Ignore deprecated MR1 properties in JobConf
+    configurationPropsToSkipCompare
+            .add(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY);
+    configurationPropsToSkipCompare
+            .add(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
+  }
+
+}


Mime
View raw message