Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id AEF34200CF0 for ; Thu, 7 Sep 2017 23:36:31 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id AD731160BEE; Thu, 7 Sep 2017 21:36:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7CFA916098B for ; Thu, 7 Sep 2017 23:36:30 +0200 (CEST) Received: (qmail 83467 invoked by uid 500); 7 Sep 2017 21:36:18 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 79870 invoked by uid 99); 7 Sep 2017 21:36:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 07 Sep 2017 21:36:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DD174F56DE; Thu, 7 Sep 2017 21:36:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Thu, 07 Sep 2017 21:36:49 -0000 Message-Id: In-Reply-To: <1ebb4b4706714ade8a78cff529bb4fae@git.apache.org> References: <1ebb4b4706714ade8a78cff529bb4fae@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [36/37] hadoop git commit: HDFS-12350. Support meta tags in configs. Contributed by Ajay Kumar. archived-at: Thu, 07 Sep 2017 21:36:31 -0000 HDFS-12350. Support meta tags in configs. Contributed by Ajay Kumar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4cd1019 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4cd1019 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4cd1019 Branch: refs/heads/HDFS-7240 Commit: a4cd101934ae5a5cad9663de872fb4ecee0d7560 Parents: 83449ab Author: Anu Engineer Authored: Thu Sep 7 12:40:09 2017 -0700 Committer: Anu Engineer Committed: Thu Sep 7 12:40:09 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/conf/Configuration.java | 116 ++++++++++++++++++- .../org/apache/hadoop/conf/CorePropertyTag.java | 37 ++++++ .../org/apache/hadoop/conf/HDFSPropertyTag.java | 41 +++++++ .../org/apache/hadoop/conf/PropertyTag.java | 30 +++++ .../org/apache/hadoop/conf/YarnPropertyTag.java | 39 +++++++ .../apache/hadoop/conf/TestConfiguration.java | 112 +++++++++++++++++- 6 files changed, 373 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4cd1019/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index a1a40b9..a339dac 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -259,7 +259,18 @@ public class Configuration implements Iterable>, */ private static final WeakHashMap REGISTRY = new WeakHashMap(); - + + /** + * Map to register all classes holding property tag enums. + */ + private static final Map + REGISTERED_TAG_CLASS = new HashMap<>(); + /** + * Map to hold properties by there tag groupings. + */ + private final Map propertyTagsMap = + new ConcurrentHashMap<>(); + /** * List of default Resources. Resources are loaded in the order of the list * entries @@ -738,6 +749,12 @@ public class Configuration implements Iterable>, public Configuration(boolean loadDefaults) { this.loadDefaults = loadDefaults; updatingResource = new ConcurrentHashMap(); + + // Register all classes holding property tags with + REGISTERED_TAG_CLASS.put("core", CorePropertyTag.class); + REGISTERED_TAG_CLASS.put("hdfs", HDFSPropertyTag.class); + REGISTERED_TAG_CLASS.put("yarn", YarnPropertyTag.class); + synchronized(Configuration.class) { REGISTRY.put(this, null); } @@ -765,6 +782,8 @@ public class Configuration implements Iterable>, this.finalParameters = Collections.newSetFromMap( new ConcurrentHashMap()); this.finalParameters.addAll(other.finalParameters); + this.REGISTERED_TAG_CLASS.putAll(other.REGISTERED_TAG_CLASS); + this.propertyTagsMap.putAll(other.propertyTagsMap); } synchronized(Configuration.class) { @@ -2823,6 +2842,10 @@ public class Configuration implements Iterable>, } else if ("source".equals(propertyAttr)) { confSource.add(StringInterner.weakIntern( reader.getAttributeValue(i))); + } else if ("tag".equals(propertyAttr)) { + //Read tags and put them in propertyTagsMap + readTagFromConfig(reader.getAttributeValue(i), confName, + confValue, confSource); } } break; @@ -2830,6 +2853,7 @@ public class Configuration implements Iterable>, case "value": case "final": case "source": + case "tag": parseToken = true; token.setLength(0); break; @@ -2911,6 +2935,13 @@ public class Configuration implements Iterable>, case "source": confSource.add(StringInterner.weakIntern(token.toString())); break; + case "tag": + if (token.length() > 0) { + //Read tags and put them in propertyTagsMap + readTagFromConfig(token.toString(), confName, + confValue, confSource); + } + break; case "include": if (fallbackAllowed && !fallbackEntered) { throw new IOException("Fetch fail on include for '" @@ -2962,6 +2993,48 @@ public class Configuration implements Iterable>, } } + private void readTagFromConfig(String attributeValue, String confName, String + confValue, List confSource) { + for (String tagStr : attributeValue.split(",")) { + tagStr = tagStr.trim(); + try { + if (confSource.size() > 0) { + for (String source : confSource) { + PropertyTag tag1 = this.getPropertyTag(tagStr, + source.split("-")[0]); + if (propertyTagsMap.containsKey(tag1)) { + propertyTagsMap.get(tag1) + .setProperty(confName, confValue); + } else { + Properties props = new Properties(); + props.setProperty(confName, confValue); + propertyTagsMap.put(tag1, props); + } + } + } else { + //If no source is set try to find tag in CorePropertyTag + if (propertyTagsMap + .containsKey(CorePropertyTag.valueOf(tagStr) + )) { + propertyTagsMap.get(CorePropertyTag.valueOf(tagStr)) + .setProperty(confName, confValue); + } else { + Properties props = new Properties(); + props.setProperty(confName, confValue); + propertyTagsMap.put(CorePropertyTag.valueOf(tagStr), + props); + } + } + } catch (IllegalArgumentException iae) { + //Log the invalid tag and continue to parse rest of the + // properties. + LOG.info("Invalid tag '" + tagStr + "' found for " + + "property:" + confName, iae); + } + + } + } + private void overlay(Properties to, Properties from) { for (Entry entry: from.entrySet()) { to.put(entry.getKey(), entry.getValue()); @@ -3438,4 +3511,45 @@ public class Configuration implements Iterable>, } return false; } + + /** + * Get all properties belonging to tag. + * @return Properties with matching properties + */ + public Properties getAllPropertiesByTag(final PropertyTag tag) { + Properties props = new Properties(); + if (propertyTagsMap.containsKey(tag)) { + props.putAll(propertyTagsMap.get(tag)); + } + return props; + } + + /** + * Get all properties belonging to list of input tags. Calls + * getAllPropertiesByTag internally. + * + * @return Properties with all matching properties + */ + public Properties getAllPropertiesByTags(final List tagList) { + Properties prop = new Properties(); + for (PropertyTag tag : tagList) { + prop.putAll(this.getAllPropertiesByTag(tag)); + } + return prop; + } + + /** + * Get Property tag Enum corresponding to given source. + * + * @param tagStr String representation of Enum + * @param group Group to which enum belongs.Ex hdfs,yarn + * @return Properties with all matching properties + */ + private PropertyTag getPropertyTag(String tagStr, String group) { + PropertyTag tag = null; + if (REGISTERED_TAG_CLASS.containsKey(group)) { + tag = (PropertyTag) Enum.valueOf(REGISTERED_TAG_CLASS.get(group), tagStr); + } + return tag; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4cd1019/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/CorePropertyTag.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/CorePropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/CorePropertyTag.java new file mode 100644 index 0000000..54a75b8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/CorePropertyTag.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*************************************************************************** + * Enum for tagging hadoop core properties according to there usage. + * CorePropertyTag implements the + * {@link org.apache.hadoop.conf.PropertyTag} interface, + ***************************************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public enum CorePropertyTag implements PropertyTag { + CORE, + REQUIRED, + PERFORMANCE, + CLIENT, + SERVER, + SECURITY, + DEBUG +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4cd1019/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/HDFSPropertyTag.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/HDFSPropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/HDFSPropertyTag.java new file mode 100644 index 0000000..02dfb86 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/HDFSPropertyTag.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*************************************************************************** + * Enum for tagging hdfs properties according to there usage or application. + * HDFSPropertyTag implements the + * {@link org.apache.hadoop.conf.PropertyTag} interface, + ***************************************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public enum HDFSPropertyTag implements PropertyTag { + HDFS, + NAMENODE, + DATANODE, + REQUIRED, + SECURITY, + KERBEROS, + PERFORMANCE, + CLIENT, + SERVER, + DEBUG, + DEPRICATED +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4cd1019/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/PropertyTag.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/PropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/PropertyTag.java new file mode 100644 index 0000000..df8d4f9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/PropertyTag.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*********************************************************** + * PropertyTag is used for creating extendable property tag Enums. + * Property tags will group related properties together. + ***********************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface PropertyTag { + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4cd1019/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/YarnPropertyTag.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/YarnPropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/YarnPropertyTag.java new file mode 100644 index 0000000..e7a9c79 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/YarnPropertyTag.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*************************************************************************** + * Enum for tagging yarn properties according to there usage or application. + * YarnPropertyTag implements the + * {@link org.apache.hadoop.conf.PropertyTag} interface, + ***************************************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public enum YarnPropertyTag implements PropertyTag { + YARN, + RESOURCEMANAGER, + SECURITY, + KERBEROS, + PERFORMANCE, + CLIENT, + REQUIRED, + SERVER, + DEBUG +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4cd1019/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index b41a807..4cd1666 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -38,6 +38,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.regex.Pattern; @@ -53,7 +54,6 @@ import static org.junit.Assert.assertArrayEquals; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration.IntegerRanges; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; @@ -135,6 +135,7 @@ public class TestConfiguration { private void endConfig() throws IOException{ out.write("\n"); + out.flush(); out.close(); } @@ -577,6 +578,34 @@ public class TestConfiguration { out.write("\n"); } + private void appendPropertyByTag(String name, String val, String tags, + String... sources) throws IOException { + appendPropertyByTag(name, val, false, tags, sources); + } + + private void appendPropertyByTag(String name, String val, boolean isFinal, + String tag, String... sources) throws IOException { + out.write(""); + out.write(""); + out.write(name); + out.write(""); + out.write(""); + out.write(val); + out.write(""); + if (isFinal) { + out.write("true"); + } + for (String s : sources) { + out.write(""); + out.write(s); + out.write(""); + } + out.write(""); + out.write(tag); + out.write(""); + out.write("\n"); + } + void appendCompactFormatProperty(String name, String val) throws IOException { appendCompactFormatProperty(name, val, false); } @@ -2215,4 +2244,85 @@ public class TestConfiguration { TestConfiguration.class.getName() }); } + + @Test + public void testGetAllPropertiesByTags() throws Exception { + + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendPropertyByTag("dfs.cblock.trace.io", "false", "DEBUG"); + appendPropertyByTag("dfs.replication", "1", "PERFORMANCE,REQUIRED"); + appendPropertyByTag("dfs.namenode.logging.level", "INFO", "CLIENT,DEBUG"); + endConfig(); + + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + conf.getProps(); + + List tagList = new ArrayList<>(); + tagList.add(CorePropertyTag.REQUIRED); + tagList.add(CorePropertyTag.PERFORMANCE); + tagList.add(CorePropertyTag.DEBUG); + tagList.add(CorePropertyTag.CLIENT); + + Properties properties = conf.getAllPropertiesByTags(tagList); + assertEq(3, properties.size()); + assertEq(true, properties.containsKey("dfs.namenode.logging.level")); + assertEq(true, properties.containsKey("dfs.replication")); + assertEq(true, properties.containsKey("dfs.cblock.trace.io")); + assertEq(false, properties.containsKey("namenode.host")); + } + + @Test + public void testGetAllPropertiesWithSourceByTags() throws Exception { + + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendPropertyByTag("dfs.cblock.trace.io", "false", "DEBUG", + "hdfs-default.xml", "core-site.xml"); + appendPropertyByTag("dfs.replication", "1", "PERFORMANCE,HDFS", + "hdfs-default.xml"); + appendPropertyByTag("yarn.resourcemanager.work-preserving-recovery" + + ".enabled", "INFO", "CLIENT,DEBUG", "yarn-default.xml", "yarn-site" + + ".xml"); + endConfig(); + + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + conf.getProps(); + + List tagList = new ArrayList<>(); + tagList.add(CorePropertyTag.REQUIRED); + + Properties properties; + properties = conf.getAllPropertiesByTags(tagList); + assertNotEquals(3, properties.size()); + + tagList.add(HDFSPropertyTag.DEBUG); + tagList.add(YarnPropertyTag.CLIENT); + tagList.add(HDFSPropertyTag.PERFORMANCE); + tagList.add(HDFSPropertyTag.HDFS); + properties = conf.getAllPropertiesByTags(tagList); + assertEq(3, properties.size()); + + assertEq(true, properties.containsKey("dfs.cblock.trace.io")); + assertEq(true, properties.containsKey("dfs.replication")); + assertEq(true, properties + .containsKey("yarn.resourcemanager.work-preserving-recovery.enabled")); + assertEq(false, properties.containsKey("namenode.host")); + + tagList.clear(); + tagList.add(HDFSPropertyTag.DEBUG); + properties = conf.getAllPropertiesByTags(tagList); + assertEq(true, properties.containsKey("dfs.cblock.trace.io")); + assertEq(false, properties.containsKey("yarn.resourcemanager" + + ".work-preserving-recovery")); + + tagList.clear(); + tagList.add(YarnPropertyTag.DEBUG); + properties = conf.getAllPropertiesByTags(tagList); + assertEq(false, properties.containsKey("dfs.cblock.trace.io")); + assertEq(true, properties.containsKey("yarn.resourcemanager" + + ".work-preserving-recovery.enabled")); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org