ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject ignite git commit: IGNITE-2206: Docs.
Date Mon, 04 Jan 2016 10:39:33 GMT
Repository: ignite
Updated Branches:
  refs/heads/ignite-2206 4b231539d -> bc6a3f124


IGNITE-2206: Docs.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bc6a3f12
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bc6a3f12
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bc6a3f12

Branch: refs/heads/ignite-2206
Commit: bc6a3f1249665ef24a9584320001d5c2ec1ebdbd
Parents: 4b23153
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Mon Jan 4 13:40:13 2016 +0400
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Mon Jan 4 13:40:13 2016 +0400

----------------------------------------------------------------------
 .../hadoop/fs/BasicHadoopFileSystemFactory.java | 31 +++++++++++++----
 .../fs/CachingHadoopFileSystemFactory.java      | 16 +++++++--
 .../hadoop/fs/HadoopFileSystemFactory.java      | 35 ++++++++++++--------
 .../fs/IgniteHadoopIgfsSecondaryFileSystem.java |  2 +-
 .../hadoop/fs/v1/IgniteHadoopFileSystem.java    |  4 +--
 .../hadoop/fs/v2/IgniteHadoopFileSystem.java    |  4 +--
 ...oopSecondaryFileSystemConfigurationTest.java |  2 +-
 .../IgniteHadoopFileSystemAbstractSelfTest.java |  2 +-
 8 files changed, 66 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
index bc22da8..dae8703 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/BasicHadoopFileSystemFactory.java
@@ -20,6 +20,7 @@ package org.apache.ignite.hadoop.fs;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
 import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
 import org.apache.ignite.internal.processors.igfs.IgfsUtils;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -36,7 +37,9 @@ import java.net.URL;
 import java.util.Arrays;
 
 /**
- * Simple Hadoop file system factory factory which delegates to {@code FileSystem.get()}.
+ * Simple Hadoop file system factory which delegates to {@code FileSystem.get()} on each
call.
+ * <p>
+ * If {@code "fs.[prefix].impl.disable.cache"} is set to {@code true}, file system instances
will be cached by Hadoop.
  */
 public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory, Externalizable,
LifecycleAware {
     /** File system URI. */
@@ -52,14 +55,14 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory,
Ex
     protected transient URI fullUri;
 
     /**
-     * Public non-arg constructor.
+     * Constructor.
      */
     public BasicHadoopFileSystemFactory() {
-        // noop
+        // No-op.
     }
 
     /** {@inheritDoc} */
-    @Override public FileSystem create(String usrName) throws IOException {
+    @Override public FileSystem get(String usrName) throws IOException {
         return create0(IgfsUtils.fixUserName(usrName));
     }
 
@@ -85,6 +88,11 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory,
Ex
 
     /**
      * Gets file system URI.
+     * <p>
+     * This URI will be used as a first argument when calling {@link FileSystem#get(URI,
Configuration, String)}.
+     * <p>
+     * If not set, default URI will be picked from file system configuration using
+     * {@link FileSystem#getDefaultUri(Configuration)} method.
      *
      * @return File system URI.
      */
@@ -93,7 +101,7 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory,
Ex
     }
 
     /**
-     * Sets file system URI.
+     * Sets file system URI. See {@link #getUri()} for more information.
      *
      * @param uri File system URI.
      */
@@ -103,6 +111,14 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory,
Ex
 
     /**
      * Gets paths to additional file system configuration files (e.g. core-site.xml).
+     * <p>
+     * Path could be either absolute or relative to {@code IGNITE_HOME} environment variable.
+     * <p>
+     * All provided paths will be loaded in the order they provided and then applied to {@link
Configuration}. It means
+     * that path order might be important in some cases.
+     * <p>
+     * <b>NOTE!</b> Factory can be serialized and transferred to other machines
where instance of
+     * {@link IgniteHadoopFileSystem} resides. Corresponding paths must exist on these machines
as well.
      *
      * @return Paths to file system configuration files.
      */
@@ -111,7 +127,8 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory,
Ex
     }
 
     /**
-     * Set paths to additional file system configuration files (e.g. core-site.xml).
+     * Set paths to additional file system configuration files (e.g. core-site.xml). See
{@link #getConfigPaths()} for
+     * more information.
      *
      * @param cfgPaths Paths to file system configuration files.
      */
@@ -126,7 +143,7 @@ public class BasicHadoopFileSystemFactory implements HadoopFileSystemFactory,
Ex
         if (cfgPaths != null) {
             for (String cfgPath : cfgPaths) {
                 if (cfgPath == null)
-                    throw new IgniteException("Configuration path cannot be null: " + Arrays.toString(cfgPaths));
+                    throw new NullPointerException("Configuration path cannot be null: "
+ Arrays.toString(cfgPaths));
                 else {
                     URL url = U.resolveIgniteUrl(cfgPath);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
index baad1fc..97064fd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.hadoop.fs;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -25,12 +26,21 @@ import org.apache.ignite.internal.processors.hadoop.fs.HadoopLazyConcurrentMap;
 import org.apache.ignite.internal.processors.igfs.IgfsUtils;
 
 import java.io.IOException;
+import java.net.URI;
 
 /**
- * Caching Hadoop file system factory. File systems are cache on per-user basis.
+ * Caching Hadoop file system factory. Caches {@link FileSystem} instances on per-user basis.
Doesn't rely on
+ * built-in Hadoop {@code FileSystem} caching mechanics. Separate {@code FileSystem} instance
is created for each
+ * user instead.
+ * <p>
+ * This makes cache instance resistant to concurrent calls to {@link FileSystem#close()}
in other parts of the user
+ * code. On the other hand, this might cause problems on some environments. E.g. if Kerberos
is enabled, a call to
+ * {@link FileSystem#get(URI, Configuration, String)} will refresh Kerberos token. But this
factory implementation
+ * calls this method only once per user what may lead to token expiration. In such cases
it makes sense to either
+ * use {@link BasicHadoopFileSystemFactory} or implement your own factory.
  */
 public class CachingHadoopFileSystemFactory extends BasicHadoopFileSystemFactory {
-    /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method.
*/
+    /** Per-user file system cache. */
     private final transient HadoopLazyConcurrentMap<String, FileSystem> cache = new
HadoopLazyConcurrentMap<>(
         new HadoopLazyConcurrentMap.ValueFactory<String, FileSystem>() {
             @Override public FileSystem createValue(String key) throws IOException {
@@ -47,7 +57,7 @@ public class CachingHadoopFileSystemFactory extends BasicHadoopFileSystemFactory
     }
 
     /** {@inheritDoc} */
-    @Override public FileSystem create(String usrName) throws IOException {
+    @Override public FileSystem get(String usrName) throws IOException {
         return cache.getOrCreate(IgfsUtils.fixUserName(usrName));
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
index ae38cc4..5ad08ab 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
@@ -17,27 +17,36 @@
 
 package org.apache.ignite.hadoop.fs;
 
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
+import org.apache.ignite.igfs.IgfsMode;
+import org.apache.ignite.lifecycle.LifecycleAware;
+
 import java.io.IOException;
 import java.io.Serializable;
-import org.apache.hadoop.fs.FileSystem;
 
 /**
- * Factory for
- *
- * Hadoop file system factory. Used to construct instance of {@link FileSystem} for Ignite.
+ * Factory for Hadoop {@link FileSystem} used by {@link IgniteHadoopIgfsSecondaryFileSystem}.
+ * <p>
+ * {@link #get(String)} method will be used whenever a call to a target {@code FileSystem}
is required.
+ * <p>
+ * It is implementation dependent whether to rely on built-in Hadoop file system cache, implement
own caching facility
+ * or doesn't cache file systems at all.
+ * <p>
+ * Concrete factory may implement {@link LifecycleAware} interface. In this case start and
stop callbacks will be
+ * performed by Ignite. You may want to implement some initialization or cleanup there.
  * <p>
- * Implementations may choose not to construct a new instance, but instead
- * return a previously created instance.
+ * Note that factory extends {@link Serializable} interface as it might be necessary to transfer
factories over the
+ * wire to {@link IgniteHadoopFileSystem} if {@link IgfsMode#PROXY} is enabled for some file
+ * system paths.
  */
 public interface HadoopFileSystemFactory extends Serializable {
     /**
-     * Creates the file system, possibly taking a cached instance.
-     * All the other data needed for the file system creation are expected to be contained
-     * in this object instance.
+     * Gets file system for the given user name.
      *
-     * @param usrName The user name
-     * @return The file system.
-     * @throws IOException On error.
+     * @param usrName User name
+     * @return File system.
+     * @throws IOException In case of error.
      */
-    public FileSystem create(String usrName) throws IOException;
+    public FileSystem get(String usrName) throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index d15a31e..9f544c1 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -518,7 +518,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
         assert !F.isEmpty(user);
 
         try {
-            return fsFactory.create(user);
+            return fsFactory.get(user);
         }
         catch (IOException ioe) {
             throw new IgniteException(ioe);

http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 4b8e989..71f6435 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -341,7 +341,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
                     ((LifecycleAware) factory).start();
 
                 try {
-                    FileSystem secFs = factory.create(user);
+                    FileSystem secFs = factory.get(user);
 
                     secondaryUri = secFs.getUri();
 
@@ -1350,6 +1350,6 @@ public class IgniteHadoopFileSystem extends FileSystem {
         if (factory == null)
             return null;
 
-        return factory.create(user);
+        return factory.get(user);
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index d3e732c..0d7de86 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -350,7 +350,7 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements
Closea
                     ((LifecycleAware) factory).start();
 
                 try {
-                    FileSystem secFs = factory.create(user);
+                    FileSystem secFs = factory.get(user);
 
                     secondaryUri = secFs.getUri();
 
@@ -1066,6 +1066,6 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements
Closea
     private FileSystem secondaryFileSystem() throws IOException{
         assert factory != null;
 
-        return factory.create(user);
+        return factory.get(user);
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
index 59bf7ee..d9b5d66 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
@@ -181,7 +181,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
 
         fac.start();
 
-        primaryFs = fac.create(null); //provider.createFileSystem(null);
+        primaryFs = fac.get(null); //provider.createFileSystem(null);
 
         primaryFsUri = primaryFs.getUri();
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/bc6a3f12/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 14ff25d..a0c781f 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -412,7 +412,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
                 return new IgniteHadoopFileSystem().getUri();
             }
         }, IllegalStateException.class,
-            "URI is null (was IgniteHadoopFileSystem properly initialized?) [closed=false]");
+            "URI is null (was IgniteHadoopFileSystem properly initialized?)");
     }
 
     /** @throws Exception If failed. */


Mime
View raw message