hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject hadoop git commit: HADOOP-15295. Remove redundant logging related to tags from Configuration. Contributed by Ajay Kumar.
Date Sat, 24 Mar 2018 04:32:25 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 24f75e097 -> 28790b81e


HADOOP-15295. Remove redundant logging related to tags from Configuration. Contributed by
Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28790b81
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28790b81
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28790b81

Branch: refs/heads/trunk
Commit: 28790b81ecb719ac26a01ad0c400ee5f1c29ccf8
Parents: 24f75e0
Author: Anu Engineer <aengineer@apache.org>
Authored: Fri Mar 23 21:17:18 2018 -0700
Committer: Anu Engineer <aengineer@apache.org>
Committed: Fri Mar 23 21:17:18 2018 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/conf/Configuration.java   | 41 ++++++++++----------
 .../apache/hadoop/conf/TestConfiguration.java   | 34 ++++------------
 2 files changed, 27 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/28790b81/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index e0a5866..25fd656a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -229,7 +229,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private static final Logger LOG_DEPRECATION =
       LoggerFactory.getLogger(
           "org.apache.hadoop.conf.Configuration.deprecation");
-  private static final Set<String> TAGS = new HashSet<>();
+  private static final Set<String> TAGS = ConcurrentHashMap.newKeySet();
 
   private boolean quietmode = true;
 
@@ -2935,7 +2935,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
         resources.set(i, ret);
       }
     }
-    this.removeUndeclaredTags(properties);
+    this.addTags(properties);
   }
   
   private Resource loadResource(Properties properties,
@@ -3183,29 +3183,28 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   }
 
   /**
-   * Removes undeclared tags and related properties from propertyTagsMap.
-   * Its required because ordering of properties in xml config files is not
-   * guaranteed.
+   * Add tags defined in HADOOP_SYSTEM_TAGS, HADOOP_CUSTOM_TAGS.
    * @param prop
    */
-  private void removeUndeclaredTags(Properties prop) {
+  public void addTags(Properties prop) {
     // Get all system tags
-    if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)){
-      String systemTags = prop.getProperty(CommonConfigurationKeys
-              .HADOOP_SYSTEM_TAGS);
-      Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
-    }
-    // Get all custom tags
-    if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
-      String customTags = prop.getProperty(CommonConfigurationKeys
-          .HADOOP_CUSTOM_TAGS);
-      Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
-    }
+    try {
+      if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)) {
+        String systemTags = prop.getProperty(CommonConfigurationKeys
+            .HADOOP_SYSTEM_TAGS);
+        Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
+      }
+      // Get all custom tags
+      if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
+        String customTags = prop.getProperty(CommonConfigurationKeys
+            .HADOOP_CUSTOM_TAGS);
+        Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
+      }
 
-    Set undeclaredTags = propertyTagsMap.keySet();
-    if (undeclaredTags.retainAll(TAGS)) {
-      LOG.info("Removed undeclared tags:");
+    } catch (Exception ex) {
+      LOG.trace("Error adding tags in configuration", ex);
     }
+
   }
 
   /**
@@ -3219,8 +3218,8 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
   private void readTagFromConfig(String attributeValue, String confName, String
       confValue, List<String> confSource) {
     for (String tagStr : attributeValue.split(",")) {
-      tagStr = tagStr.trim();
       try {
+        tagStr = tagStr.trim();
         // Handle property with no/null value
         if (confValue == null) {
           confValue = "";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/28790b81/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 8f54464..b0bb0d7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -27,7 +27,6 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStreamWriter;
-import java.io.PrintStream;
 import java.io.StringWriter;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
@@ -2392,33 +2391,14 @@ public class TestConfiguration {
 
   @Test
   public void testInvalidTags() throws Exception {
-    PrintStream output = System.out;
-    try {
-      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(bytes));
-
-      out = new BufferedWriter(new FileWriter(CONFIG));
-      startConfig();
-      appendPropertyByTag("dfs.cblock.trace.io", "false", "MYOWNTAG,TAG2");
-      endConfig();
-
-      Path fileResource = new Path(CONFIG);
-      conf.addResource(fileResource);
-      conf.getProps();
-
-      List<String> tagList = new ArrayList<>();
-      tagList.add("REQUIRED");
-      tagList.add("MYOWNTAG");
-      tagList.add("TAG2");
+    Path fileResource = new Path(CONFIG);
+    conf.addResource(fileResource);
+    conf.getProps();
 
-      Properties properties = conf.getAllPropertiesByTags(tagList);
-      assertEq(0, properties.size());
-      assertFalse(properties.containsKey("dfs.cblock.trace.io"));
-      assertFalse(bytes.toString().contains("Invalid tag "));
-      assertFalse(bytes.toString().contains("Tag"));
-    } finally {
-      System.setOut(output);
-    }
+    assertFalse(conf.isPropertyTag("BADTAG"));
+    assertFalse(conf.isPropertyTag("CUSTOM_TAG"));
+    assertTrue(conf.isPropertyTag("DEBUG"));
+    assertTrue(conf.isPropertyTag("HDFS"));
   }
 
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message