ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From akuznet...@apache.org
Subject [38/48] ignite git commit: IGNITE-3343: IGFS: Secondary file system is not queried for statuses during MKDIRS and CREATE operations. This closes #896.
Date Wed, 03 Aug 2016 11:29:20 GMT
IGNITE-3343: IGFS: Secondary file system is not queried for statuses during MKDIRS and CREATE
operations. This closes #896.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4f8d1822
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4f8d1822
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4f8d1822

Branch: refs/heads/ignite-3443
Commit: 4f8d1822520afcffd1c97c3e643f16ff1bd85513
Parents: de86843
Author: Ivan Veselovskiy <iveselovskiy@gridgain.com>
Authored: Tue Aug 2 11:11:24 2016 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Tue Aug 2 11:11:24 2016 +0300

----------------------------------------------------------------------
 .../org/apache/ignite/IgniteFileSystem.java     |  3 +-
 .../igfs/secondary/IgfsSecondaryFileSystem.java |  4 +-
 .../processors/igfs/IgfsCreateResult.java       |  2 +-
 .../internal/processors/igfs/IgfsImpl.java      | 63 ++++++-----------
 .../processors/igfs/IgfsMetaManager.java        | 74 +++++---------------
 .../processors/igfs/IgfsModeResolver.java       | 31 ++++++--
 .../internal/processors/igfs/IgfsPaths.java     |  6 +-
 .../IgfsSecondaryFileSystemCreateContext.java   |  3 +
 .../internal/processors/igfs/IgfsUtils.java     | 12 +++-
 .../processors/igfs/IgfsAbstractSelfTest.java   | 34 ++++++++-
 .../igfs/IgfsDualAbstractSelfTest.java          | 13 ++++
 .../igfs/IgfsModeResolverSelfTest.java          | 51 ++++++++++++--
 .../hadoop/fs/v1/IgniteHadoopFileSystem.java    |  7 +-
 .../hadoop/fs/v2/IgniteHadoopFileSystem.java    |  7 +-
 .../IgniteHadoopFileSystemAbstractSelfTest.java |  1 -
 .../hadoop/HadoopAbstractMapReduceTest.java     | 13 +++-
 16 files changed, 196 insertions(+), 128 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
index bbc5bea..f9aeb8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
@@ -362,8 +362,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
      * will not be affected. Other properties will be added or overwritten. Passed properties
with {@code null} values
      * will be removed from the stored properties or ignored if they don't exist in the file
info.
      * <p>
-     * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties
will be propagated
-     * to the secondary file system:
+     * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes with Hadoop secondary
file system only the following properties will be updated:
      * <ul>
      * <li>{@code usrName} - file owner name;</li>
      * <li>{@code grpName} - file owner group;</li>

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
index 3f124eb..4d9d255 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -43,8 +43,8 @@ public interface IgfsSecondaryFileSystem {
      * will not be affected. Other properties will be added or overwritten. Passed properties
with {@code null} values
      * will be removed from the stored properties or ignored if they don't exist in the file
info.
      * <p>
-     * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties
will be propagated
-     * to the secondary file system:
+     * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes with Hadoop secondary
file system only the
+     * following properties will be updated on the secondary file system:
      * <ul>
      * <li>{@code usrName} - file owner name;</li>
      * <li>{@code grpName} - file owner group;</li>

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
index 0b09e02..eb1bc9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
@@ -23,7 +23,7 @@ import org.jetbrains.annotations.Nullable;
 import java.io.OutputStream;
 
 /**
- * IGFS file create result.
+ * IGFS file create or append result.
  */
 public class IgfsCreateResult {
     /** File info in the primary file system. */

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 0d1ffc1..e1f8e61 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -248,7 +249,7 @@ public final class IgfsImpl implements IgfsEx {
             }
         }
 
-        modeRslvr = new IgfsModeResolver(dfltMode, IgfsUtils.preparePathModes(dfltMode, modes));
+        modeRslvr = new IgfsModeResolver(dfltMode, modes);
 
         Object secondaryFsPayload = null;
 
@@ -816,12 +817,14 @@ public final class IgfsImpl implements IgfsEx {
                     }
                 }
 
-                IgniteUuid fileId = meta.fileId(path);
+                if (!IgfsUtils.isDualMode(mode) || modeRslvr.hasPrimaryChild(path)) {
+                    IgniteUuid fileId = meta.fileId(path);
 
-                if (fileId != null)
-                    files.addAll(meta.directoryListing(fileId).keySet());
-                else if (mode == PRIMARY)
-                    throw new IgfsPathNotFoundException("Failed to list files (path not found):
" + path);
+                    if (fileId != null)
+                        files.addAll(meta.directoryListing(fileId).keySet());
+                    else if (mode == PRIMARY)
+                        throw new IgfsPathNotFoundException("Failed to list files (path not
found): " + path);
+                }
 
                 return F.viewReadOnly(files, new C1<String, IgfsPath>() {
                     @Override public IgfsPath apply(String e) {
@@ -846,7 +849,7 @@ public final class IgfsImpl implements IgfsEx {
 
                 IgfsMode mode = resolveMode(path);
 
-                Collection<IgfsFile> files = new HashSet<>();
+                Set<IgfsFile> files = new HashSet<>();
 
                 if (IgfsUtils.isDualMode(mode)) {
                     assert secondaryFs != null;
@@ -859,6 +862,9 @@ public final class IgfsImpl implements IgfsEx {
 
                             files.add(impl);
                         }
+
+                        if (!modeRslvr.hasPrimaryChild(path))
+                            return files;
                     }
                     catch (Exception e) {
                         U.error(log, "List files in DUAL mode failed [path=" + path + ']',
e);
@@ -1345,29 +1351,6 @@ public final class IgfsImpl implements IgfsEx {
         return fut;
     }
 
-    /**
-     * Get file descriptor for specified path.
-     *
-     * @param path Path to file.
-     * @return Detailed file descriptor or {@code null}, if file does not exist.
-     * @throws IgniteCheckedException If failed.
-     */
-    @Nullable private FileDescriptor getFileDescriptor(IgfsPath path) throws IgniteCheckedException
{
-        assert path != null;
-
-        List<IgniteUuid> ids = meta.idsForPath(path);
-
-        IgfsEntryInfo fileInfo = meta.info(ids.get(ids.size() - 1));
-
-        if (fileInfo == null)
-            return null; // File does not exist.
-
-        // Resolve parent ID for removed file.
-        IgniteUuid parentId = ids.size() >= 2 ? ids.get(ids.size() - 2) : null;
-
-        return new FileDescriptor(parentId, path.name(), fileInfo.id(), fileInfo.isFile());
-    }
-
     /** {@inheritDoc} */
     @Override public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver
rslvr,
         Collection<IgfsPath> paths, @Nullable T arg) {
@@ -1519,20 +1502,16 @@ public final class IgfsImpl implements IgfsEx {
 
             case DUAL_SYNC:
             case DUAL_ASYNC:
-                info = meta.infoForPath(path);
-
-                if (info == null) {
-                    try {
-                        IgfsFile status = secondaryFs.info(path);
+                try {
+                    IgfsFile status = secondaryFs.info(path);
 
-                        if (status != null)
-                            return new IgfsFileImpl(status, data.groupBlockSize());
-                    }
-                    catch (Exception e) {
-                        U.error(log, "File info operation in DUAL mode failed [path=" + path
+ ']', e);
+                    if (status != null)
+                        return new IgfsFileImpl(status, data.groupBlockSize());
+                }
+                catch (Exception e) {
+                    U.error(log, "File info operation in DUAL mode failed [path=" + path
+ ']', e);
 
-                        throw e;
-                    }
+                    throw e;
                 }
 
                 break;

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index d891b38..0d14a3a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -3051,29 +3051,19 @@ public class IgfsMetaManager extends IgfsManager {
                             if (secondaryCtx != null) {
                                 secondaryOut = secondaryCtx.create();
 
-                                IgfsFile secondaryFile = secondaryCtx.info();
-
-                                if (secondaryFile == null)
-                                    throw fsException("Failed to open output stream to the
file created in " +
-                                        "the secondary file system because it no longer exists:
" + path);
-                                else if (secondaryFile.isDirectory())
-                                    throw fsException("Failed to open output stream to the
file created in " +
-                                        "the secondary file system because the path points
to a directory: " + path);
-
-                                newAccessTime = secondaryFile.accessTime();
-                                newModificationTime = secondaryFile.modificationTime();
-                                newProps = secondaryFile.properties();
-                                newLen = secondaryFile.length();
-                                newBlockSize = secondaryFile.blockSize();
+                                newAccessTime = 0L;
+                                newModificationTime = 0L;
+                                newProps = null;
                             }
                             else {
                                 newAccessTime = System.currentTimeMillis();
                                 newModificationTime = newAccessTime;
                                 newProps = fileProps;
-                                newLen = 0L;
-                                newBlockSize = blockSize;
                             }
 
+                            newLen = 0L;
+                            newBlockSize = blockSize;
+
                             IgfsEntryInfo newInfo = invokeAndGet(overwriteId,
                                 new IgfsMetaFileCreateProcessor(newAccessTime, newModificationTime,
newProps,
                                     newBlockSize, affKey, newLockId, evictExclude, newLen));
@@ -3254,18 +3244,9 @@ public class IgfsMetaManager extends IgfsManager {
             Map<String, String> props;
 
             if (secondaryCtx != null) {
-                IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(lastCreatedPath);
-
-                if (secondaryInfo == null)
-                    throw new IgfsException("Failed to perform operation because secondary
file system path was " +
-                        "modified concurrently: " + lastCreatedPath);
-                else if (secondaryInfo.isFile())
-                    throw new IgfsException("Failed to perform operation because secondary
file system entity is " +
-                        "not directory: " + lastCreatedPath);
-
-                accessTime = secondaryInfo.accessTime();
-                modificationTime = secondaryInfo.modificationTime();
-                props = secondaryInfo.properties();
+                accessTime = 0L;
+                modificationTime = 0L;
+                props = null;
             }
             else {
                 accessTime = curTime;
@@ -3293,18 +3274,9 @@ public class IgfsMetaManager extends IgfsManager {
             Map<String, String> props;
 
             if (secondaryCtx != null) {
-                IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(pathIds.path());
-
-                if (secondaryInfo == null)
-                    throw new IgfsException("Failed to perform operation because secondary
file system path was " +
-                        "modified concurrnetly: " + pathIds.path());
-                else if (secondaryInfo.isFile())
-                    throw new IgfsException("Failed to perform operation because secondary
file system entity is " +
-                        "not directory: " + lastCreatedPath);
-
-                accessTime = secondaryInfo.accessTime();
-                modificationTime = secondaryInfo.modificationTime();
-                props = secondaryInfo.properties();
+                accessTime = 0L;
+                modificationTime = 0L;
+                props = null;
             }
             else {
                 accessTime = curTime;
@@ -3322,29 +3294,19 @@ public class IgfsMetaManager extends IgfsManager {
             int newBlockSize;
 
             if (secondaryCtx != null) {
-                IgfsFile secondaryFile = secondaryCtx.info();
-
-                if (secondaryFile == null)
-                    throw fsException("Failed to open output stream to the file created in
" +
-                        "the secondary file system because it no longer exists: " + pathIds.path());
-                else if (secondaryFile.isDirectory())
-                    throw fsException("Failed to open output stream to the file created in
" +
-                        "the secondary file system because the path points to a directory:
" + pathIds.path());
-
-                newAccessTime = secondaryFile.accessTime();
-                newModificationTime = secondaryFile.modificationTime();
-                newProps = secondaryFile.properties();
-                newLen = secondaryFile.length();
-                newBlockSize = secondaryFile.blockSize();
+                newAccessTime = 0L;
+                newModificationTime = 0L;
+                newProps = null;
             }
             else {
                 newAccessTime = curTime;
                 newModificationTime = curTime;
                 newProps = fileProps;
-                newLen = 0L;
-                newBlockSize = blockSize;
             }
 
+            newLen = 0L;
+            newBlockSize = blockSize;
+
             procMap.put(curId, new IgfsMetaFileCreateProcessor(newAccessTime, newModificationTime,
newProps,
                 newBlockSize, affKey, createFileLockId(false), evictExclude, newLen));
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
index 907051f..33b835f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
@@ -17,9 +17,12 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
-import java.util.Collections;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.igfs.IgfsMode;
 import org.apache.ignite.igfs.IgfsPath;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap;
@@ -42,18 +45,24 @@ public class IgfsModeResolver {
     /** Cached modes per path. */
     private Map<IgfsPath, IgfsMode> modesCache;
 
+    /** Set to store parent dual paths that have primary children. */
+    private final Set<IgfsPath> dualParentsWithPrimaryChildren;
+
     /**
      * Constructor
      *
      * @param dfltMode Default IGFS mode.
      * @param modes List of configured modes. The order is significant as modes are added
in order of occurrence.
      */
-    public IgfsModeResolver(IgfsMode dfltMode, @Nullable List<T2<IgfsPath, IgfsMode>>
modes) {
+    public IgfsModeResolver(IgfsMode dfltMode, @Nullable ArrayList<T2<IgfsPath, IgfsMode>>
modes)
+            throws IgniteCheckedException {
         assert dfltMode != null;
 
         this.dfltMode = dfltMode;
 
-        this.modes = modes;
+        this.dualParentsWithPrimaryChildren = new HashSet<>();
+
+        this.modes = IgfsUtils.preparePathModes(dfltMode, modes, dualParentsWithPrimaryChildren);
 
         if (modes != null)
             modesCache = new GridBoundedConcurrentLinkedHashMap<>(MAX_PATH_CACHE);
@@ -94,10 +103,20 @@ public class IgfsModeResolver {
     }
 
     /**
-     * @return Unmodifiable copy of properly ordered modes prefixes
+     * @return Copy of properly ordered modes prefixes
      *  or {@code null} if no modes set.
      */
-    @Nullable public List<T2<IgfsPath, IgfsMode>> modesOrdered() {
-        return modes != null ? Collections.unmodifiableList(modes) : null;
+    @Nullable public ArrayList<T2<IgfsPath, IgfsMode>> modesOrdered() {
+        return modes != null ? new ArrayList<>(modes) : null;
+    }
+
+    /**
+     * Answers if the given path has an immediate child of PRIMARY mode.
+     *
+     * @param path The path to query.
+     * @return If the given path has an immediate child of PRIMARY mode.
+     */
+    public boolean hasPrimaryChild(IgfsPath path) {
+        return dualParentsWithPrimaryChildren.contains(path);
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index 4a79259..2bdb23b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -48,7 +48,7 @@ public class IgfsPaths implements Externalizable {
     private IgfsMode dfltMode;
 
     /** Path modes. */
-    private List<T2<IgfsPath, IgfsMode>> pathModes;
+    private ArrayList<T2<IgfsPath, IgfsMode>> pathModes;
 
     /**
      * Empty constructor required by {@link Externalizable}.
@@ -65,7 +65,7 @@ public class IgfsPaths implements Externalizable {
      * @param pathModes Path modes.
      * @throws IgniteCheckedException If failed.
      */
-    public IgfsPaths(Object payload, IgfsMode dfltMode, @Nullable List<T2<IgfsPath,
IgfsMode>> pathModes)
+    public IgfsPaths(Object payload, IgfsMode dfltMode, @Nullable ArrayList<T2<IgfsPath,
IgfsMode>> pathModes)
         throws IgniteCheckedException {
         this.dfltMode = dfltMode;
         this.pathModes = pathModes;
@@ -91,7 +91,7 @@ public class IgfsPaths implements Externalizable {
     /**
      * @return Path modes.
      */
-    @Nullable public List<T2<IgfsPath, IgfsMode>> pathModes() {
+    @Nullable public ArrayList<T2<IgfsPath, IgfsMode>> pathModes() {
         return pathModes;
     }
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
index 1c0efd6..788efe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
@@ -28,6 +28,7 @@ import java.util.Map;
 
 /**
  * Context for secondary file system create request.
+ * Note that it is never used for dual mode append operation.
  */
 public class IgfsSecondaryFileSystemCreateContext {
     /** File system. */
@@ -68,6 +69,8 @@ public class IgfsSecondaryFileSystemCreateContext {
      */
     public IgfsSecondaryFileSystemCreateContext(IgfsSecondaryFileSystem fs, IgfsPath path,
boolean overwrite,
         boolean simpleCreate, @Nullable Map<String, String> props, short replication,
long blockSize, int bufSize) {
+        assert fs != null;
+
         this.fs = fs;
         this.path = path;
         this.overwrite = overwrite;

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
index 03b7611..b9788f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.Set;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -66,6 +67,7 @@ import java.util.concurrent.ThreadLocalRandom;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_RETRIES_COUNT;
 import static org.apache.ignite.igfs.IgfsMode.DUAL_ASYNC;
 import static org.apache.ignite.igfs.IgfsMode.DUAL_SYNC;
+import static org.apache.ignite.igfs.IgfsMode.PRIMARY;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGFS;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -819,11 +821,13 @@ public class IgfsUtils {
      *
      * @param dfltMode The root mode. Must always be not null.
      * @param modes The subdirectory modes.
+     * @param dualParentsContainingPrimaryChildren The set to store parents into.
      * @return Descending list of filtered and checked modes.
-     * @throws IgniteCheckedException On error or
+     * @throws IgniteCheckedException On error.
      */
     public static ArrayList<T2<IgfsPath, IgfsMode>> preparePathModes(final IgfsMode
dfltMode,
-        @Nullable List<T2<IgfsPath, IgfsMode>> modes) throws IgniteCheckedException
{
+        @Nullable List<T2<IgfsPath, IgfsMode>> modes, Set<IgfsPath> dualParentsContainingPrimaryChildren)
+        throws IgniteCheckedException {
         if (modes == null)
             return null;
 
@@ -857,6 +861,10 @@ public class IgfsUtils {
                     // Add to the 1st position (deep first).
                     resModes.add(0, mode);
 
+                    // Store primary paths inside dual paths in separate collection:
+                    if (mode.getValue() == PRIMARY)
+                        dualParentsContainingPrimaryChildren.add(mode.getKey().parent());
+
                     break;
                 }
             }

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 384da95..fd3f9b6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -245,6 +245,13 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     }
 
     /**
+     * @return Relaxed consistency flag.
+     */
+    protected boolean initializeDefaultPathModes() {
+        return false;
+    }
+
+    /**
      * @return Client flag.
      */
     protected boolean client() {
@@ -369,6 +376,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         igfsCfg.setSequentialReadsBeforePrefetch(SEQ_READS_BEFORE_PREFETCH);
         igfsCfg.setRelaxedConsistency(relaxedConsistency());
 
+        igfsCfg.setInitializeDefaultPathModes(initializeDefaultPathModes());
+
         CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
 
         dataCacheCfg.setName("dataCache");
@@ -1071,17 +1080,36 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @throws Exception If failed.
      */
     public void testRootPropertiesPersistAfterFormat() throws Exception {
-        igfs.update(new IgfsPath("/"), Collections.singletonMap("foo", "moo"));
+        if (dual && !(igfsSecondaryFileSystem instanceof IgfsSecondaryFileSystemImpl))
{
+            // In case of Hadoop dual mode only user name, group name, and permission properties
are updated,
+            // an arbitrary named property is just ignored:
+            checkRootPropertyUpdate("foo", "moo", null);
+            checkRootPropertyUpdate(IgfsUtils.PROP_PERMISSION, "0777", "0777");
+        }
+        else {
+            checkRootPropertyUpdate("foo", "moo", "moo");
+            checkRootPropertyUpdate(IgfsUtils.PROP_PERMISSION, "0777", "0777");
+        }
+    }
+
+    /**
+     *
+     * @throws Exception
+     */
+    private void checkRootPropertyUpdate(String prop, String setVal, String expGetVal) throws
Exception {
+        final IgfsPath rootPath = new IgfsPath("/");
+
+        igfs.update(rootPath, Collections.singletonMap(prop, setVal));
 
         igfs.format();
 
-        IgfsFile file = igfs.info(new IgfsPath("/"));
+        IgfsFile file = igfs.info(rootPath);
 
         assert file != null;
 
         Map<String,String> props = file.properties();
 
-        assertEquals("moo", props.get("foo"));
+        assertEquals(expGetVal, props.get(prop));
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
index 124bec6..92e1178 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
@@ -55,6 +55,12 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
     }
 
+    /** {@inheritDoc} */
+    @Override protected boolean initializeDefaultPathModes() {
+        // Enable default modes in order to test various modes.
+        return true;
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -68,6 +74,13 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         for (IgfsPath p : paths)
             assert igfs.exists(p);
 
+        assert igfs.modeResolver().resolveMode(gg) == mode;
+        assert igfs.modeResolver().resolveMode(new IgfsPath(gg, "sync")) == IgfsMode.DUAL_SYNC;
+        assert igfs.modeResolver().resolveMode(new IgfsPath(gg, "async")) == IgfsMode.DUAL_ASYNC;
+        assert igfs.modeResolver().resolveMode(new IgfsPath(gg, "primary")) == IgfsMode.PRIMARY;
+        assert !igfsSecondary.exists("/ignite/primary"); // PRIMARY mode path must exist
in upper level fs only.
+
+        // All the child paths of "/ignite/" must be visible in listings:
         assert igfs.listFiles(gg).size() == 3;
         assert igfs.listPaths(gg).size() == 3;
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolverSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolverSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolverSelfTest.java
index f3bb516..bd7e413 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolverSelfTest.java
@@ -17,8 +17,11 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import junit.framework.TestCase;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.igfs.IgfsMode;
@@ -38,8 +41,9 @@ public class IgfsModeResolverSelfTest extends TestCase {
 
     /** {@inheritDoc} */
     @Override protected void setUp() throws Exception {
-        reslvr = new IgfsModeResolver(DUAL_SYNC, Arrays.asList(new T2<>(new IgfsPath("/a/b/c/d"),
PROXY), new T2<>
-            (new IgfsPath("/a/P/"), PRIMARY), new T2<>(new IgfsPath("/a/b/"), DUAL_ASYNC)));
+        reslvr = new IgfsModeResolver(DUAL_SYNC, new ArrayList<>(Arrays.asList(new
T2<>(
+            new IgfsPath("/a/b/c/d"), PROXY), new T2<>(new IgfsPath("/a/P/"), PRIMARY),
new T2<>(new IgfsPath("/a/b/"),
+            DUAL_ASYNC))));
     }
 
     /**
@@ -90,7 +94,7 @@ public class IgfsModeResolverSelfTest extends TestCase {
         try {
             IgfsUtils.preparePathModes(DUAL_SYNC, Arrays.asList(
                 new T2<>(new IgfsPath("/a/"), PRIMARY),
-                new T2<>(new IgfsPath("/a/b/"), DUAL_ASYNC)));
+                new T2<>(new IgfsPath("/a/b/"), DUAL_ASYNC)), new HashSet<IgfsPath>());
 
             fail("IgniteCheckedException expected");
         }
@@ -102,7 +106,8 @@ public class IgfsModeResolverSelfTest extends TestCase {
         for (IgfsMode m: IgfsMode.values()) {
             if (m != IgfsMode.PRIMARY) {
                 try {
-                    IgfsUtils.preparePathModes(PRIMARY, Arrays.asList(new T2<>(new
IgfsPath("/a/"), DUAL_ASYNC)));
+                    IgfsUtils.preparePathModes(PRIMARY, Arrays.asList(new T2<>(new
IgfsPath("/a/"), DUAL_ASYNC)),
+                        new HashSet<IgfsPath>());
 
                     fail("IgniteCheckedException expected");
                 }
@@ -117,7 +122,7 @@ public class IgfsModeResolverSelfTest extends TestCase {
             new T2<>(new IgfsPath("/a"), PRIMARY),
             new T2<>(new IgfsPath("/c/d/"), PRIMARY),
             new T2<>(new IgfsPath("/c/d/e/f"), PRIMARY)
-        ));
+        ), new HashSet<IgfsPath>());
         assertNotNull(modes);
         assertEquals(2, modes.size());
         assertEquals(modes, Arrays.asList(
@@ -130,7 +135,7 @@ public class IgfsModeResolverSelfTest extends TestCase {
             new T2<>(new IgfsPath("/a/b"), DUAL_ASYNC),
             new T2<>(new IgfsPath("/a/b/c"), DUAL_SYNC),
             new T2<>(new IgfsPath("/a/b/c/d"), DUAL_ASYNC)
-        ));
+        ), new HashSet<IgfsPath>());
         assertNotNull(modes);
         assertEquals(modes.size(), 3);
         assertEquals(modes, Arrays.asList(
@@ -139,4 +144,38 @@ public class IgfsModeResolverSelfTest extends TestCase {
             new T2<>(new IgfsPath("/a/b"), DUAL_ASYNC)
         ));
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDualParentsWithPrimaryChild() throws Exception {
+        Set<IgfsPath> set = new HashSet<>();
+
+        IgfsUtils.preparePathModes(DUAL_SYNC, Arrays.asList(
+            new T2<>(new IgfsPath("/a/b"), DUAL_ASYNC),
+            new T2<>(new IgfsPath("/a/b/c"), PRIMARY),
+            new T2<>(new IgfsPath("/a/b/x/y"), PRIMARY),
+            new T2<>(new IgfsPath("/a/b/x/z"), PRIMARY),
+            new T2<>(new IgfsPath("/m"), PRIMARY)
+        ), set);
+        assertEquals(set, new HashSet<IgfsPath>() {{
+            add(new IgfsPath("/a/b"));
+            add(new IgfsPath("/a/b/x"));
+            add(new IgfsPath("/"));
+        }});
+
+        set = new HashSet<>();
+
+        IgfsUtils.preparePathModes(DUAL_ASYNC, Arrays.asList(
+            new T2<>(new IgfsPath("/a/b/x/y/z"), PRIMARY),
+            new T2<>(new IgfsPath("/a/b/c"), PRIMARY),
+            new T2<>(new IgfsPath("/a/k"), PRIMARY),
+            new T2<>(new IgfsPath("/a/z"), PRIMARY)
+        ), set);
+        assertEquals(set, new HashSet<IgfsPath>() {{
+            add(new IgfsPath("/a/b"));
+            add(new IgfsPath("/a"));
+            add(new IgfsPath("/a/b/x/y"));
+        }});
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 83991aa..a06129e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -309,7 +309,12 @@ public class IgniteHadoopFileSystem extends FileSystem {
             else
                 clientLog = IgfsLogger.disabledLogger();
 
-            modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
+            try {
+                modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
+            }
+            catch (IgniteCheckedException ice) {
+                throw new IOException(ice);
+            }
 
             boolean initSecondary = paths.defaultMode() == PROXY;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index cc02d05..bd8ed2d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -318,7 +318,12 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements
Closea
             else
                 clientLog = IgfsLogger.disabledLogger();
 
-            modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
+            try {
+                modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());
+            }
+            catch (IgniteCheckedException ice) {
+                throw new IOException(ice);
+            }
 
             boolean initSecondary = paths.defaultMode() == PROXY;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 083ee54..f793ec3 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -48,7 +48,6 @@ import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;

http://git-wip-us.apache.org/repos/asf/ignite/blob/4f8d1822/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
index ef886e4..3731213 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
@@ -41,6 +41,7 @@ import org.apache.ignite.configuration.HadoopConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.hadoop.fs.IgniteHadoopFileSystemCounterWriter;
 import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
+import org.apache.ignite.igfs.IgfsFile;
 import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper;
 import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
 import org.apache.ignite.igfs.IgfsMode;
@@ -123,8 +124,16 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest
{
      * @param p The path.
      * @return The owner.
      */
-    private static String getOwner(IgfsEx i, IgfsPath p) {
-        return i.info(p).property(IgfsUtils.PROP_USER_NAME);
+    private static String getOwner(final IgfsEx i, final IgfsPath p) {
+        return IgfsUserContext.doAs(USER, new IgniteOutClosure<String>() {
+            @Override public String apply() {
+                IgfsFile f = i.info(p);
+
+                assert f != null;
+
+                return f.property(IgfsUtils.PROP_USER_NAME);
+            }
+        });
     }
 
     /**


Mime
View raw message