hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject [18/50] [abbrv] hadoop git commit: HADOOP-14399. Configuration does not correctly XInclude absolute file URIs. Contributed by Jonathan Eagles
Date Wed, 31 May 2017 22:42:53 GMT
HADOOP-14399. Configuration does not correctly XInclude absolute file URIs.
Contributed by Jonathan Eagles


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ba9704e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ba9704e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ba9704e

Branch: refs/heads/YARN-1011
Commit: 1ba9704eec22c75f8aec653ee15eb6767b5a7f4b
Parents: 1a56a3d
Author: Steve Loughran <stevel@apache.org>
Authored: Thu May 25 14:59:33 2017 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Thu May 25 14:59:33 2017 +0100

----------------------------------------------------------------------
 .../org/apache/hadoop/conf/Configuration.java   | 37 ++++++++++++--------
 .../apache/hadoop/conf/TestConfiguration.java   | 23 +++++++++---
 2 files changed, 42 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba9704e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 2ac52cb..1a6679b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2714,6 +2714,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       StringBuilder token = new StringBuilder();
       String confName = null;
       String confValue = null;
+      String confInclude = null;
       boolean confFinal = false;
       boolean fallbackAllowed = false;
       boolean fallbackEntered = false;
@@ -2757,7 +2758,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
             break;
           case "include":
             // Determine href for xi:include
-            String confInclude = null;
+            confInclude = null;
             attrCount = reader.getAttributeCount();
             for (int i = 0; i < attrCount; i++) {
               String attrName = reader.getAttributeLocalName(i);
@@ -2776,18 +2777,25 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
               Resource classpathResource = new Resource(include, name);
               loadResource(properties, classpathResource, quiet);
             } else {
-              File href = new File(confInclude);
-              if (!href.isAbsolute()) {
-                // Included resources are relative to the current resource
-                File baseFile = new File(name).getParentFile();
-                href = new File(baseFile, href.getPath());
+              URL url;
+              try {
+                url = new URL(confInclude);
+                url.openConnection().connect();
+              } catch (IOException ioe) {
+                File href = new File(confInclude);
+                if (!href.isAbsolute()) {
+                  // Included resources are relative to the current resource
+                  File baseFile = new File(name).getParentFile();
+                  href = new File(baseFile, href.getPath());
+                }
+                if (!href.exists()) {
+                  // Resource errors are non-fatal iff there is 1 xi:fallback
+                  fallbackAllowed = true;
+                  break;
+                }
+                url = href.toURI().toURL();
               }
-              if (!href.exists()) {
-                // Resource errors are non-fatal iff there is 1 xi:fallback
-                fallbackAllowed = true;
-                break;
-              }
-              Resource uriResource = new Resource(href.toURI().toURL(), name);
+              Resource uriResource = new Resource(url, name);
               loadResource(properties, uriResource, quiet);
             }
             break;
@@ -2828,8 +2836,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
             break;
           case "include":
             if (fallbackAllowed && !fallbackEntered) {
-              throw new IOException("Fetch fail on include with no "
-                  + "fallback while loading '" + name + "'");
+              throw new IOException("Fetch fail on include for '"
+                  + confInclude + "' with no fallback while loading '"
+                  + name + "'");
             }
             fallbackAllowed = false;
             fallbackEntered = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba9704e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 6f4c26c..0c66470 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -29,6 +29,7 @@ import java.io.OutputStreamWriter;
 import java.io.StringWriter;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -61,6 +62,9 @@ public class TestConfiguration extends TestCase {
   final static String CONFIG = new File("./test-config-TestConfiguration.xml").getAbsolutePath();
   final static String CONFIG2 = new File("./test-config2-TestConfiguration.xml").getAbsolutePath();
   final static String CONFIG_FOR_ENUM = new File("./test-config-enum-TestConfiguration.xml").getAbsolutePath();
+  final static String CONFIG_FOR_URI = "file://"
+      + new File("./test-config-uri-TestConfiguration.xml").getAbsolutePath();
+
   private static final String CONFIG_MULTI_BYTE = new File(
     "./test-config-multi-byte-TestConfiguration.xml").getAbsolutePath();
   private static final String CONFIG_MULTI_BYTE_SAVED = new File(
@@ -85,6 +89,7 @@ public class TestConfiguration extends TestCase {
     new File(CONFIG).delete();
     new File(CONFIG2).delete();
     new File(CONFIG_FOR_ENUM).delete();
+    new File(new URI(CONFIG_FOR_URI)).delete();
     new File(CONFIG_MULTI_BYTE).delete();
     new File(CONFIG_MULTI_BYTE_SAVED).delete();
   }
@@ -516,13 +521,21 @@ public class TestConfiguration extends TestCase {
     appendProperty("a","b");
     appendProperty("c","d");
     endConfig();
+    File fileUri = new File(new URI(CONFIG_FOR_URI));
+    out=new BufferedWriter(new FileWriter(fileUri));
+    startConfig();
+    appendProperty("e", "f");
+    appendProperty("g", "h");
+    endConfig();
 
     out=new BufferedWriter(new FileWriter(CONFIG));
     startConfig();
     startInclude(CONFIG2);
     endInclude();
-    appendProperty("e","f");
-    appendProperty("g","h");
+    startInclude(CONFIG_FOR_URI);
+    endInclude();
+    appendProperty("i", "j");
+    appendProperty("k", "l");
     endConfig();
 
     // verify that the includes file contains all properties
@@ -530,8 +543,10 @@ public class TestConfiguration extends TestCase {
     conf.addResource(fileResource);
     assertEquals(conf.get("a"), "b"); 
     assertEquals(conf.get("c"), "d"); 
-    assertEquals(conf.get("e"), "f"); 
-    assertEquals(conf.get("g"), "h"); 
+    assertEquals(conf.get("e"), "f");
+    assertEquals(conf.get("g"), "h");
+    assertEquals(conf.get("i"), "j");
+    assertEquals(conf.get("k"), "l");
     tearDown();
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message