commons-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rwhitc...@apache.org
Subject svn commit: r1720416 - in /commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs: HdfsFileSystem.java HdfsFileSystemConfigBuilder.java
Date Wed, 16 Dec 2015 18:59:37 GMT
Author: rwhitcomb
Date: Wed Dec 16 18:59:37 2015
New Revision: 1720416

URL: http://svn.apache.org/viewvc?rev=1720416&view=rev
Log:
VFS-586:  Add several additional ways to specify an alternate HDFS configuration.

Motivation:  the previous changes in this area allowed specifying the name of
one resource (in the CLASSPATH) for an alternate configuration.  Further use
with this feature showed the need for being able to specify more than one
resource (possibly), and to be able to load configurations from local files
as well as basically any way that HDFS configuration parameters can be
specified (according to the HDFS APIs).

Changes:  therefore, the API in HdfsFileSystemConfigBuilder was greatly
extended to allow:
a) Multiple resource names to be given;
b) File system paths to be specified (one or many);
c) URLs to be listed (multiple if desired) so configuration can be loaded
   from a web application;
d) Another HDFS Configuration object (one of these);
e) Any miscellaneous InputStream (only one of these).

And the configuration step in HdfsFileSystem was enhanced to check for
all these possible sources of configuration information and add the
available ones to the HDFS "Configuration" object during initialization.

One of the existing APIs was changed:  getConfigName -> getConfigNames
(with a corresponding change to the return type:  String -> String[]).

Note: I have not been able to figure out a good unit test for this yet,
but it is working quite well in our application.

Modified:
    commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
    commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java

Modified: commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
URL: http://svn.apache.org/viewvc/commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java?rev=1720416&r1=1720415&r2=1720416&view=diff
==============================================================================
--- commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
(original)
+++ commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
Wed Dec 16 18:59:37 2015
@@ -16,8 +16,10 @@
  */
 package org.apache.commons.vfs2.provider.hdfs;
 
+import java.io.InputStream;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.net.URL;
 import java.net.URLDecoder;
 import java.util.Collection;
 
@@ -110,14 +112,51 @@ public class HdfsFileSystem extends Abst
             {
                 final String hdfsUri = name.getRootURI();
                 final HdfsFileSystemConfigBuilder builder = HdfsFileSystemConfigBuilder.getInstance();
-                final String configName = builder.getConfigName(getFileSystemOptions());
+                final FileSystemOptions options = getFileSystemOptions();
+                final String[] configNames = builder.getConfigNames(options);
+                final Path[] configPaths = builder.getConfigPaths(options);
+                final URL[] configURLs = builder.getConfigURLs(options);
+                final InputStream configStream = builder.getConfigInputStream(options);
+                final Configuration configConfiguration = builder.getConfigConfiguration(options);
 
                 final Configuration conf = new Configuration(true);
                 conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);
-                if (configName != null)
+
+                // Load any alternate configuration parameters that may have been specified
+                // no matter where they might come from
+                if (configNames != null)
+                {
+                    for (String configName : configNames)
+                    {
+                        log.debug("Adding HDFS configuration resource: " + configName);
+                        conf.addResource(configName);
+                    }
+                }
+                if (configPaths != null)
+                {
+                    for (Path path : configPaths)
+                    {
+                        log.debug("Adding HDFS configuration path: " + path);
+                        conf.addResource(path);
+                    }
+                }
+                if (configURLs != null)
+                {
+                    for (URL url : configURLs)
+                    {
+                        log.debug("Adding HDFS configuration URL: " + url);
+                        conf.addResource(url);
+                    }
+                }
+                if (configStream != null)
+                {
+                    log.debug("Adding HDFS configuration stream");
+                    conf.addResource(configStream);
+                }
+                if (configConfiguration != null)
                 {
-                    log.debug("Adding HDFS configuration file: " + configName);
-                    conf.addResource(configName);
+                    log.debug("Adding HDFS configuration object");
+                    conf.addResource(configConfiguration);
                 }
 
                 try

Modified: commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java
URL: http://svn.apache.org/viewvc/commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java?rev=1720416&r1=1720415&r2=1720416&view=diff
==============================================================================
--- commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java
(original)
+++ commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java
Wed Dec 16 18:59:37 2015
@@ -16,9 +16,14 @@
  */
 package org.apache.commons.vfs2.provider.hdfs;
 
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
 import org.apache.commons.vfs2.FileSystem;
 import org.apache.commons.vfs2.FileSystemConfigBuilder;
 import org.apache.commons.vfs2.FileSystemOptions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
 
 /**
  * Configuration settings for the HdfsFileSystem.
@@ -28,7 +33,11 @@ import org.apache.commons.vfs2.FileSyste
 public final class HdfsFileSystemConfigBuilder extends FileSystemConfigBuilder
 {
     private static final HdfsFileSystemConfigBuilder BUILDER = new HdfsFileSystemConfigBuilder();
-    private static final String KEY_CONFIG_NAME = "configName";
+    private static final String KEY_CONFIG_NAMES  = "configNames";
+    private static final String KEY_CONFIG_PATHS  = "configPaths";
+    private static final String KEY_CONFIG_URLS   = "configURLs";
+    private static final String KEY_CONFIG_STREAM = "configStream";
+    private static final String KEY_CONFIG_CONF   = "configConf";
 
     private HdfsFileSystemConfigBuilder()
     {
@@ -53,34 +62,252 @@ public final class HdfsFileSystemConfigB
     }
 
     /**
-     * Get name of alternate configuration file.
+     * Get names of alternate configuration resources.
      *
-     * @return resource name of alternate configuration file or null.
+     * @return resource name list of alternate configurations or {@code null}.
      * @param opts The FileSystemOptions.
      * @see #setConfigName(FileSystemOptions, String)
      */
-    public String getConfigName(final FileSystemOptions opts)
+    public String[] getConfigNames(final FileSystemOptions opts)
     {
-        return this.getString(opts, KEY_CONFIG_NAME);
+        String names = this.getString(opts, KEY_CONFIG_NAMES);
+        return names == null || names.isEmpty() ? null : names.split(",");
     }
 
     /**
-     * Sets the name of configuration file to be loaded after the defaults.
+     * Sets the name of configuration resource to be loaded after the defaults.
      * <p>
-     * Specifies the name of a config file to override any specific HDFS settings.
+     * Specifies the name of a config resource to override any specific HDFS settings.
      * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(String)}
      * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY,
url)}.
      * <p>
      * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
      * property in order to access HDFS files stored in an AWS installation (from outside
their
      * firewall). There are other possible uses too.
+     * <p>
+     * This method may be called multiple times and all the specified resources will be loaded
+     * in the order they were specified.
+     * <p>
+     * Note also, that if a list of names is provided, separated by commas ({@code ","}),
that
+     * this will work the same as calling this method a number of times with just one name
each.
      *
      * @param opts The FileSystemOptions to modify.
-     * @param name resource name of additional configuration file or null.
+     * @param name resource name of additional configuration or {@code null} to unset all
the
+     * values set so far.
+     * @see #getConfigNames
      */
     public void setConfigName(final FileSystemOptions opts, final String name)
     {
-        this.setParam(opts, KEY_CONFIG_NAME, name);
+        if (name == null || name.isEmpty())
+        {
+            this.setParam(opts, KEY_CONFIG_NAMES, null);
+        }
+        else
+        {
+            String previousNames = this.getString(opts, KEY_CONFIG_NAMES);
+            if (previousNames == null || previousNames.isEmpty())
+            {
+                this.setParam(opts, KEY_CONFIG_NAMES, name);
+            }
+            else
+            {
+                this.setParam(opts, KEY_CONFIG_NAMES, previousNames + "," + name);
+            }
+        }
+    }
+
+    /**
+     * Get paths of alternate configuration file system files.
+     *
+     * @return list of full paths of alternate configuration files or {@code null}.
+     * @param opts The FileSystemOptions.
+     * @see #setConfigPath(FileSystemOptions, Path)
+     */
+    public Path[] getConfigPaths(final FileSystemOptions opts)
+    {
+        String pathNames = this.getString(opts, KEY_CONFIG_PATHS);
+        if (pathNames == null || pathNames.isEmpty())
+        {
+            return null;
+        }
+        String[] paths = pathNames.split(",");
+        Path[] realPaths = new Path[paths.length];
+        for (int i = 0; i < paths.length; i++)
+        {
+            realPaths[i] = new Path(paths[i]);
+        }
+        return realPaths;
+    }
+
+    /**
+     * Sets the full path of configuration file to be loaded after the defaults.
+     * <p>
+     * Specifies the path of a local file system config file to override any specific HDFS
settings.
+     * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Path)}
+     * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY,
url)}.
+     * <p>
+     * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
+     * property in order to access HDFS files stored in an AWS installation (from outside
their
+     * firewall). There are other possible uses too.
+     * <p>
+     * This method may be called multiple times and all the specified resources will be loaded
+     * in the order they were specified.
+     *
+     * @param opts The FileSystemOptions to modify.
+     * @param path full path of additional configuration file (local file system) or {@code
null}
+     * to unset all the path values set so far.
+     */
+    public void setConfigPath(final FileSystemOptions opts, final Path path)
+    {
+        if (path == null)
+        {
+            this.setParam(opts, KEY_CONFIG_PATHS, null);
+        }
+        else
+        {
+            String previousPathNames = this.getString(opts, KEY_CONFIG_PATHS);
+            if (previousPathNames == null || previousPathNames.isEmpty())
+            {
+                this.setParam(opts, KEY_CONFIG_PATHS, path.toString());
+            }
+            else
+            {
+                this.setParam(opts, KEY_CONFIG_PATHS, previousPathNames + "," + path.toString());
+            }
+        }
+    }
+
+    /**
+     * Get URLs of alternate configurations.
+     *
+     * @return list of alternate configuration URLs or {@code null}.
+     * @param opts The FileSystemOptions.
+     * @see #setConfigURL(FileSystemOptions, URL)
+     */
+    public URL[] getConfigURLs(final FileSystemOptions opts)
+    {
+        try
+        {
+            String urlNames = this.getString(opts, KEY_CONFIG_URLS);
+            if (urlNames == null || urlNames.isEmpty())
+            {
+                return null;
+            }
+            String[] urls = urlNames.split(",");
+            URL[] realURLs = new URL[urls.length];
+            for (int i = 0; i < urls.length; i++)
+            {
+                realURLs[i] = new URL(urls[i]);
+            }
+            return realURLs;
+        }
+        catch (MalformedURLException mue)
+        {
+            // This should never happen because we save it in the proper form
+        }
+        return null;
+    }
+
+    /**
+     * Sets the URL of configuration file to be loaded after the defaults.
+     * <p>
+     * Specifies the URL of a config file to override any specific HDFS settings.
+     * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(URL)}
+     * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY,
url)}.
+     * <p>
+     * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
+     * property in order to access HDFS files stored in an AWS installation (from outside
their
+     * firewall). There are other possible uses too.
+     * <p>
+     * This method may be called multiple times and all the specified resources will be loaded
+     * in the order they were specified.
+     *
+     * @param opts The FileSystemOptions to modify.
+     * @param url URL of additional configuration file or {@code null} to unset all the URL
+     * values set so far.
+     */
+    public void setConfigURL(final FileSystemOptions opts, final URL url)
+    {
+        if (url == null)
+        {
+            this.setParam(opts, KEY_CONFIG_URLS, null);
+        }
+        else
+        {
+            String previousURLNames = this.getString(opts, KEY_CONFIG_URLS);
+            if (previousURLNames == null || previousURLNames.isEmpty())
+            {
+                this.setParam(opts, KEY_CONFIG_URLS, url.toString());
+            }
+            else
+            {
+                this.setParam(opts, KEY_CONFIG_URLS, previousURLNames + "," + url.toString());
+            }
+        }
+    }
+
+    /**
+     * Get alternate configuration input stream.
+     *
+     * @return alternate configuration input stream or {@code null}.
+     * @param opts The FileSystemOptions.
+     * @see #setConfigInputStream(FileSystemOptions, InputStream)
+     */
+    public InputStream getConfigInputStream(final FileSystemOptions opts)
+    {
+        return (InputStream)this.getParam(opts, KEY_CONFIG_STREAM);
+    }
+
+    /**
+     * Sets the input stream of configuration file to be loaded after the defaults.
+     * <p>
+     * Specifies an input stream connected to a config file to override any specific HDFS
settings.
+     * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(InputStream)}
+     * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY,
url)}.
+     * <p>
+     * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
+     * property in order to access HDFS files stored in an AWS installation (from outside
their
+     * firewall). There are other possible uses too.
+     *
+     * @param opts The FileSystemOptions to modify.
+     * @param inputStream input stream of additional configuration file or {@code null} to
unset
+     * the configuration input stream previously set up.
+     */
+    public void setConfigInputStream(final FileSystemOptions opts, final InputStream inputStream)
+    {
+        this.setParam(opts, KEY_CONFIG_STREAM, inputStream);
+    }
+
+    /**
+     * Get alternate configuration object.
+     *
+     * @return alternate configuration object or {@code null}.
+     * @param opts The FileSystemOptions.
+     * @see #setConfigConfiguration(FileSystemOptions, Configuration)
+     */
+    public Configuration getConfigConfiguration(final FileSystemOptions opts)
+    {
+        return (Configuration)this.getParam(opts, KEY_CONFIG_CONF);
+    }
+
+    /**
+     * Sets the configuration object to be loaded after the defaults.
+     * <p>
+     * Specifies an already initialized configuration object to override any specific HDFS
settings.
+     * The property will be passed on to {@code org.apache.hadoop.conf.Configuration#addResource(Configuration)}
+     * after the URL was set as the default name with: {@code Configuration#set(FileSystem.FS_DEFAULT_NAME_KEY,
url)}.
+     * <p>
+     * One use for this is to set a different value for the {@code dfs.client.use.datanode.hostname}
+     * property in order to access HDFS files stored in an AWS installation (from outside
their
+     * firewall). There are other possible uses too.
+     *
+     * @param opts The FileSystemOptions to modify.
+     * @param configuration additional configuration object or {@code null} to unset any
configuration
+     * object previously set.
+     */
+    public void setConfigConfiguration(final FileSystemOptions opts, final Configuration
configuration)
+    {
+        this.setParam(opts, KEY_CONFIG_CONF, configuration);
     }
 
 }



Mime
View raw message