Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 68733105CD for ; Tue, 5 May 2015 21:50:15 +0000 (UTC) Received: (qmail 83010 invoked by uid 500); 5 May 2015 21:50:15 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 82944 invoked by uid 500); 5 May 2015 21:50:15 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 82935 invoked by uid 99); 5 May 2015 21:50:15 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 May 2015 21:50:15 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EFCD5E13CA; Tue, 5 May 2015 21:50:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: rkanter@apache.org To: common-commits@hadoop.apache.org Message-Id: <0ac2d9da07d74d11adadf06781be2a1e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter) Date: Tue, 5 May 2015 21:50:14 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 0100b1550 -> 9809a16d3 MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9809a16d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9809a16d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9809a16d Branch: refs/heads/trunk Commit: 9809a16d3c8068beccbf0106e99c7ede6ba11e0f Parents: 0100b15 Author: Robert Kanter Authored: Mon May 4 17:48:10 2015 -0700 Committer: Robert Kanter Committed: Tue May 5 14:43:28 2015 -0700 ---------------------------------------------------------------------- .../conf/TestConfigurationFieldsBase.java | 58 ++++++++++++++- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapred/TestMapreduceConfigFields.java | 76 ++++++++++++++++++++ 3 files changed, 135 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java index c3fe3a3..2e4d8b1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java @@ -147,6 +147,12 @@ public abstract class TestConfigurationFieldsBase { private Set xmlFieldsMissingInConfiguration = null; /** + * Member variable for debugging base class operation + */ + protected boolean configDebug = false; + protected boolean xmlDebug = false; + + /** * Abstract method to be used by subclasses for initializing base * members. */ @@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase { HashMap retVal = new HashMap(); // Setup regexp for valid properties - String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$"; + String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$"; Pattern p = Pattern.compile(propRegex); // Iterate through class member variables int totalFields = 0; String value; for (Field f : fields) { + if (configDebug) { + System.out.println("Field: " + f); + } // Filter out anything that isn't "public static final" if (!Modifier.isStatic(f.getModifiers()) || !Modifier.isPublic(f.getModifiers()) || @@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase { } catch (IllegalAccessException iaException) { continue; } + if (configDebug) { + System.out.println(" Value: " + value); + } // Special Case: Detect and ignore partial properties (ending in x) // or file properties (ending in .xml) if (value.endsWith(".xml") || @@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase { // something like: blah.blah2(.blah3.blah4...) Matcher m = p.matcher(value); if (!m.find()) { + if (configDebug) { + System.out.println(" Passes Regex: false"); + } continue; } + if (configDebug) { + System.out.println(" Passes Regex: true"); + } // Save member variable/value as hash - retVal.put(value,f.getName()); + if (!retVal.containsKey(value)) { + retVal.put(value,f.getName()); + } else { + if (configDebug) { + System.out.println("ERROR: Already found key for property " + value); + } + } } return retVal; @@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase { // Ignore known xml props if (xmlPropsToSkipCompare != null) { if (xmlPropsToSkipCompare.contains(key)) { + if (xmlDebug) { + System.out.println(" Skipping Full Key: " + key); + } continue; } } @@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase { } } if (skipPrefix) { + if (xmlDebug) { + System.out.println(" Skipping Prefix Key: " + key); + } continue; } if (conf.onlyKeyExists(key)) { retVal.put(key,null); + if (xmlDebug) { + System.out.println(" XML Key,Null Value: " + key); + } } else { String value = conf.get(key); if (value!=null) { retVal.put(key,entry.getValue()); + if (xmlDebug) { + System.out.println(" XML Key,Valid Value: " + key); + } } } kvItr.remove(); @@ -312,6 +348,10 @@ public abstract class TestConfigurationFieldsBase { // Create class member/value map configurationMemberVariables = new HashMap(); + if (configDebug) { + System.out.println("Reading configuration classes"); + System.out.println(""); + } for (Class c : configurationClasses) { Field[] fields = c.getDeclaredFields(); Map memberMap = @@ -320,9 +360,23 @@ public abstract class TestConfigurationFieldsBase { configurationMemberVariables.putAll(memberMap); } } + if (configDebug) { + System.out.println(""); + System.out.println("====="); + System.out.println(""); + } // Create XML key/value map + if (xmlDebug) { + System.out.println("Reading XML property files"); + System.out.println(""); + } xmlKeyValueMap = extractPropertiesFromXml(xmlFilename); + if (xmlDebug) { + System.out.println(""); + System.out.println("====="); + System.out.println(""); + } // Find class members not in the XML file configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 002fbe6..99621cb 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -304,6 +304,9 @@ Release 2.8.0 - UNRELEASED mapreduce.tasktracker.taskmemorymanager.monitoringinterval. (J.Andreina via aajisaka) + MAPREDUCE-6192. Create unit test to automatically compare + MR related classes and mapred-default.xml (rchiang via rkanter) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/9809a16d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java new file mode 100644 index 0000000..7f18714 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce; + +import java.util.HashSet; + +import org.apache.hadoop.conf.TestConfigurationFieldsBase; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.ShuffleHandler; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; + +/** + * Unit test class to compare the following MR Configuration classes: + *

+ * {@link org.apache.hadoop.mapreduce.MRJobConfig} + * {@link org.apache.hadoop.mapreduce.MRConfig} + * {@link org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig} + * {@link org.apache.hadoop.mapred.ShuffleHandler} + * {@link org.apache.hadoop.mapreduce.lib.output.FileOutputFormat} + * {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat} + * {@link org.apache.hadoop.mapreduce.Job} + * {@link org.apache.hadoop.mapreduce.lib.input.NLineInputFormat} + * {@link org.apache.hadoop.mapred.JobConf} + *

+ * against mapred-default.xml for missing properties. Currently only + * throws an error if the class is missing a property. + *

+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase} + * for how this class works. + */ +public class TestMapreduceConfigFields extends TestConfigurationFieldsBase { + + @SuppressWarnings("deprecation") + @Override + public void initializeMemberVariables() { + xmlFilename = new String("mapred-default.xml"); + configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class, + JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class, + FileInputFormat.class, Job.class, NLineInputFormat.class, + JobConf.class, FileOutputCommitter.class }; + + // Initialize used variables + configurationPropsToSkipCompare = new HashSet(); + + // Set error modes + errorIfMissingConfigProps = true; + errorIfMissingXmlProps = false; + + // Ignore deprecated MR1 properties in JobConf + configurationPropsToSkipCompare + .add(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY); + configurationPropsToSkipCompare + .add(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY); + } + +}