ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From anovi...@apache.org
Subject [03/20] ignite git commit: IGNITE-1515: Fixed delete.
Date Thu, 01 Oct 2015 09:23:04 GMT
IGNITE-1515: Fixed delete.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/54bb7d76
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/54bb7d76
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/54bb7d76

Branch: refs/heads/ignite-1168
Commit: 54bb7d760d9c1261a64fcf3d0be858be7b14444e
Parents: be906e3
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Wed Sep 30 10:05:19 2015 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Wed Sep 30 10:05:19 2015 +0300

----------------------------------------------------------------------
 .../java/org/apache/ignite/igfs/IgfsPath.java   |   2 +-
 .../internal/processors/igfs/IgfsFileInfo.java  |   2 +-
 .../internal/processors/igfs/IgfsImpl.java      |  57 +----
 .../processors/igfs/IgfsMetaManager.java        | 251 +++++++++++++++----
 .../ignite/igfs/IgfsFragmentizerSelfTest.java   |   2 +-
 .../processors/igfs/IgfsAbstractSelfTest.java   |  80 ++++--
 .../igfs/IgfsMetaManagerSelfTest.java           |   6 -
 .../processors/igfs/IgfsMetricsSelfTest.java    |   2 +-
 .../processors/igfs/IgfsProcessorSelfTest.java  |  29 +--
 .../igfs/UniversalFileSystemAdapter.java        |   1 -
 .../processors/hadoop/igfs/HadoopIgfsUtils.java |  36 +++
 ...oopFileSystemUniversalFileSystemAdapter.java |   4 +-
 .../HadoopIgfs20FileSystemAbstractSelfTest.java |   7 +-
 .../IgniteHadoopFileSystemAbstractSelfTest.java |   5 +-
 14 files changed, 345 insertions(+), 139 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
index a99c1ee..fb0621c 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsPath.java
@@ -50,7 +50,7 @@ public final class IgfsPath implements Comparable<IgfsPath>, Externalizable
{
     private static final char SLASH_CHAR = '/';
 
     /** The directory separator. */
-    private static final String SLASH = "/";
+    public static final String SLASH = "/";
 
     /** URI representing this path. Should never change after object creation or de-serialization.
*/
     private String path;

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
index 116d585..8564500 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java
@@ -216,7 +216,7 @@ public final class IgfsFileInfo implements Externalizable {
      * @param listing New directory listing.
      * @param old Old file info.
      */
-    IgfsFileInfo(Map<String, IgfsListingEntry> listing, IgfsFileInfo old) {
+    IgfsFileInfo(@Nullable Map<String, IgfsListingEntry> listing, IgfsFileInfo old)
{
         this(old.isDirectory(), old.id, old.blockSize, old.len, old.affKey, listing, old.props,
old.fileMap(),
             old.lockId, false, old.accessTime, old.modificationTime, old.evictExclude());
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 0dd0307..d5ba95f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -53,7 +53,6 @@ import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.IgfsEvent;
 import org.apache.ignite.igfs.IgfsBlockLocation;
-import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException;
 import org.apache.ignite.igfs.IgfsFile;
 import org.apache.ignite.igfs.IgfsInvalidPathException;
 import org.apache.ignite.igfs.IgfsMetrics;
@@ -712,6 +711,9 @@ public final class IgfsImpl implements IgfsEx {
                 if (log.isDebugEnabled())
                     log.debug("Deleting file [path=" + path + ", recursive=" + recursive
+ ']');
 
+                if (IgfsPath.SLASH.equals(path.toString()))
+                    return false;
+
                 IgfsMode mode = resolveMode(path);
 
                 Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path);
@@ -721,8 +723,11 @@ public final class IgfsImpl implements IgfsEx {
                 FileDescriptor desc = getFileDescriptor(path);
 
                 if (childrenModes.contains(PRIMARY)) {
-                    if (desc != null)
-                        res = delete0(desc, path.parent(), recursive);
+                    if (desc != null) {
+                        IgniteUuid deletedId = meta.softDelete(path, recursive);
+
+                        res = deletedId != null;
+                    }
                     else if (mode == PRIMARY)
                         checkConflictWithPrimary(path);
                 }
@@ -750,48 +755,6 @@ public final class IgfsImpl implements IgfsEx {
         });
     }
 
-    /**
-     * Internal procedure for (optionally) recursive file and directory deletion.
-     *
-     * @param desc File descriptor of file or directory to delete.
-     * @param parentPath Parent path. If specified, events will be fired for each deleted
file
-     *      or directory. If not specified, events will not be fired.
-     * @param recursive Recursive deletion flag.
-     * @return {@code True} if file was successfully deleted. If directory is not empty and
-     *      {@code recursive} flag is false, will return {@code false}.
-     * @throws IgniteCheckedException In case of error.
-     */
-    private boolean delete0(FileDescriptor desc, @Nullable IgfsPath parentPath, boolean recursive)
-        throws IgniteCheckedException {
-        IgfsPath curPath = parentPath == null ? new IgfsPath() : new IgfsPath(parentPath,
desc.fileName);
-
-        if (desc.isFile) {
-            deleteFile(curPath, desc, true);
-
-            return true;
-        }
-        else {
-            if (recursive) {
-                meta.softDelete(desc.parentId, desc.fileName, desc.fileId);
-
-                return true;
-            }
-            else {
-                Map<String, IgfsListingEntry> infoMap = meta.directoryListing(desc.fileId);
-
-                if (F.isEmpty(infoMap)) {
-                    deleteFile(curPath, desc, true);
-
-                    return true;
-                }
-                else
-                    // Throw exception if not empty and not recursive.
-                    throw new IgfsDirectoryNotEmptyException("Failed to remove directory
(directory is not empty " +
-                        "and recursive flag is not set)");
-            }
-        }
-    }
-
     /** {@inheritDoc} */
     @Override public void mkdirs(IgfsPath path) {
         mkdirs(path, null);
@@ -1454,7 +1417,7 @@ public final class IgfsImpl implements IgfsEx {
      */
     IgniteInternalFuture<?> formatAsync() {
         try {
-            IgniteUuid id = meta.softDelete(null, null, ROOT_ID);
+            IgniteUuid id = meta.format();
 
             if (id == null)
                 return new GridFinishedFuture<Object>();
@@ -1526,6 +1489,8 @@ public final class IgfsImpl implements IgfsEx {
      * @throws IgniteCheckedException If failed.
      */
     @Nullable private FileDescriptor getFileDescriptor(IgfsPath path) throws IgniteCheckedException
{
+        assert path != null;
+
         List<IgniteUuid> ids = meta.fileIds(path);
 
         IgfsFileInfo fileInfo = meta.info(ids.get(ids.size() - 1));

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index d283b64..bb6404c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -34,6 +34,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedSet;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.CountDownLatch;
@@ -95,6 +96,20 @@ import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_REA
  */
 @SuppressWarnings("all")
 public class IgfsMetaManager extends IgfsManager {
+    /** Comparator for Id sorting. */
+    private static final Comparator<IgniteUuid> PATH_ID_SORTING_COMPARATOR
+            = new Comparator<IgniteUuid>() {
+        @Override public int compare(IgniteUuid u1, IgniteUuid u2) {
+            if (u1 == u2)
+                return 0;
+
+            if (u1 == null)
+                return -1;
+
+            return u1.compareTo(u2);
+        }
+    };
+
     /** IGFS configuration. */
     private FileSystemConfiguration cfg;
 
@@ -376,6 +391,7 @@ public class IgfsMetaManager extends IgfsManager {
 
     /**
      * Gets file info by its ID.
+     * NB: this method is used both in Tx and out of Tx.
      *
      * @param fileId File ID to get details for.
      * @return File info.
@@ -593,12 +609,39 @@ public class IgfsMetaManager extends IgfsManager {
     }
 
     /**
+     * Answers if the collection is sorted.
+     *
+     * @param col The collection to check.
+     * @param <T> The type of the collection elements.
+     * @return If the collection sorted.
+     */
+    private static <T extends Comparable<T>> boolean isSorted(Collection<T>
col) {
+        T prev = null;
+
+        for (T t: col) {
+            if (t == null)
+                throw new NullPointerException("Collections should not contain nulls");
+
+            if (prev != null && prev.compareTo(t) > 0)
+                return false; // disordered.
+
+            prev = t;
+        }
+
+        return true;
+    }
+
+    /**
      * Lock file IDs.
+     *
      * @param fileIds File IDs (sorted).
      * @return Map with lock info.
      * @throws IgniteCheckedException If failed.
      */
     private Map<IgniteUuid, IgfsFileInfo> lockIds(Collection<IgniteUuid> fileIds)
throws IgniteCheckedException {
+        assert isSorted(fileIds);
+        assert validTxState(true);
+
         if (log.isDebugEnabled())
             log.debug("Locking file ids: " + fileIds);
 
@@ -608,19 +651,34 @@ public class IgfsMetaManager extends IgfsManager {
         if (log.isDebugEnabled())
             log.debug("Locked file ids: " + fileIds);
 
-        // Force root ID always exist in cache.
-        if (fileIds.contains(ROOT_ID) && !map.containsKey(ROOT_ID)) {
-            IgfsFileInfo info = new IgfsFileInfo();
+        // Force root & trash IDs always exist in cache.
+        addInfoIfNeeded(fileIds, map, ROOT_ID);
+        addInfoIfNeeded(fileIds, map, TRASH_ID);
 
-            id2InfoPrj.putIfAbsent(ROOT_ID, info);
+        // Returns detail's map for locked IDs.
+        return map;
+    }
 
-            map = new GridLeanMap<>(map);
+    /**
+     * Adds FileInfo into the cache if it is requested in fileIds and is not present in the
map.
+     *
+     * @param fileIds A list that may contain the id.
+     * @param map The map that may not contain the id.
+     * @param id The id to check.
+     * @throws IgniteCheckedException On error.
+     */
+    private void addInfoIfNeeded(Collection<IgniteUuid> fileIds, Map<IgniteUuid,
IgfsFileInfo> map, IgniteUuid id) throws IgniteCheckedException {
+        assert validTxState(true);
 
-            map.put(ROOT_ID, info);
-        }
+        if (fileIds.contains(id) && !map.containsKey(id)) {
+            IgfsFileInfo info = new IgfsFileInfo(id);
 
-        // Returns detail's map for locked IDs.
-        return map;
+            assert info.listing() != null;
+
+            id2InfoPrj.putIfAbsent(id, info);
+
+            map.put(id, info);
+        }
     }
 
     /**
@@ -779,13 +837,11 @@ public class IgfsMetaManager extends IgfsManager {
             log.debug("Locking parent id [parentId=" + parentId + ", fileName=" + fileName
+ ", newFileInfo=" +
                 newFileInfo + ']');
 
-        validTxState(true);
+        assert validTxState(true);
 
         // Lock only parent file ID.
         IgfsFileInfo parentInfo = info(parentId);
 
-        assert validTxState(true);
-
         if (parentInfo == null)
             throw fsException(new IgfsPathNotFoundException("Failed to lock parent directory
(not found): " + parentId));
 
@@ -798,8 +854,6 @@ public class IgfsMetaManager extends IgfsManager {
 
         IgfsListingEntry entry = parentListing.get(fileName);
 
-        assert validTxState(true);
-
         if (entry != null)
             return entry.fileId();
 
@@ -832,18 +886,7 @@ public class IgfsMetaManager extends IgfsManager {
                 List<IgniteUuid> srcPathIds = fileIds(srcPath);
                 List<IgniteUuid> dstPathIds = fileIds(dstPath);
 
-                final Set<IgniteUuid> allIds = new TreeSet<>(new Comparator<IgniteUuid>()
{
-                    @Override
-                    public int compare(IgniteUuid u1, IgniteUuid u2) {
-                        if (u1 == u2)
-                            return 0;
-
-                        if (u1 == null)
-                            return -1;
-
-                        return u1.compareTo(u2);
-                    }
-                });
+                final Set<IgniteUuid> allIds = new TreeSet<>(PATH_ID_SORTING_COMPARATOR);
 
                 allIds.addAll(srcPathIds);
 
@@ -1009,6 +1052,7 @@ public class IgfsMetaManager extends IgfsManager {
     private void moveNonTx(IgniteUuid fileId, @Nullable String srcFileName, IgniteUuid srcParentId,
String destFileName,
         IgniteUuid destParentId) throws IgniteCheckedException {
         assert validTxState(true);
+
         assert fileId != null;
         assert srcFileName != null;
         assert srcParentId != null;
@@ -1026,8 +1070,6 @@ public class IgfsMetaManager extends IgfsManager {
         // Lock file ID and parent IDs for this transaction.
         Map<IgniteUuid, IgfsFileInfo> infoMap = lockIds(srcParentId, fileId, destParentId);
 
-        validTxState(true);
-
         IgfsFileInfo srcInfo = infoMap.get(srcParentId);
 
         if (srcInfo == null)
@@ -1196,6 +1238,66 @@ public class IgfsMetaManager extends IgfsManager {
     }
 
     /**
+     * Deletes (moves to TRASH) all elements under the root folder.
+     *
+     * @return The new Id if the artificially created folder containing all former root
+     * elements moved to TRASH folder.
+     * @throws IgniteCheckedException On error.
+     */
+    IgniteUuid format() throws IgniteCheckedException {
+        if (busyLock.enterBusy()) {
+            try {
+                assert validTxState(false);
+
+                final IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ);
+
+                try {
+                    // NB: We may lock root because its id is less than any other id:
+                    final IgfsFileInfo rootInfo = lockIds(ROOT_ID, TRASH_ID).get(ROOT_ID);
+
+                    assert rootInfo != null;
+
+                    Map<String, IgfsListingEntry> rootListingMap = rootInfo.listing();
+
+                    assert rootListingMap != null;
+
+                    if (rootListingMap.isEmpty())
+                        return null; // Root is empty, nothing to do.
+
+                    // Construct new info and move locked entries from root to it.
+                    Map<String, IgfsListingEntry> transferListing = new HashMap<>(rootListingMap);
+
+                    IgfsFileInfo newInfo = new IgfsFileInfo(transferListing);
+
+                    id2InfoPrj.put(newInfo.id(), newInfo);
+
+                    // Add new info to trash listing.
+                    id2InfoPrj.invoke(TRASH_ID, new UpdateListing(newInfo.id().toString(),
+                        new IgfsListingEntry(newInfo), false));
+
+                    // Remove listing entries from root.
+                    // Note that root directory properties and other attributes are preserved:
+                    id2InfoPrj.put(ROOT_ID, new IgfsFileInfo(null/*listing*/, rootInfo));
+
+                    tx.commit();
+
+                    delWorker.signal();
+
+                    return newInfo.id();
+                }
+                finally {
+                    tx.close();
+                }
+            }
+            finally {
+                busyLock.leaveBusy();
+            }
+        }
+        else
+            throw new IllegalStateException("Failed to perform format because Grid is stopping.");
+    }
+
+    /**
      * Move path to the trash directory.
      *
      * @param parentId Parent ID.
@@ -1204,26 +1306,86 @@ public class IgfsMetaManager extends IgfsManager {
      * @return ID of an entry located directly under the trash directory.
      * @throws IgniteCheckedException If failed.
      */
-    IgniteUuid softDelete(@Nullable IgniteUuid parentId, @Nullable String pathName, IgniteUuid
pathId) throws IgniteCheckedException {
+    IgniteUuid softDelete(final IgfsPath path, final boolean recursive) throws IgniteCheckedException
{
         if (busyLock.enterBusy()) {
             try {
                 assert validTxState(false);
 
-                IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ);
+                final SortedSet<IgniteUuid> allIds = new TreeSet<>(PATH_ID_SORTING_COMPARATOR);
+
+                List<IgniteUuid> pathIdList = fileIds(path);
+
+                assert pathIdList.size() > 1;
+
+                final IgniteUuid victimId = pathIdList.get(pathIdList.size() - 1);
+
+                assert !TRASH_ID.equals(victimId) : "TRASH does not have path, it cannot
ever be deletion victim.";
+                assert !ROOT_ID.equals(victimId); // root deletion is prevented in earlier
stages.
+
+                allIds.addAll(pathIdList);
+
+                if (allIds.remove(null))
+                    return null; // A fragment of the path no longer exists.
+
+                boolean added = allIds.add(TRASH_ID);
+                assert added;
+
+                final IgniteInternalTx tx = metaCache.txStartEx(PESSIMISTIC, REPEATABLE_READ);
 
                 try {
-                    if (parentId == null)
-                        lockIds(pathId, TRASH_ID);
-                    else
-                        lockIds(parentId, pathId, TRASH_ID);
+                    final Map<IgniteUuid, IgfsFileInfo> infoMap = lockIds(allIds);
+
+                    // Directory starure was changed concurrently, so the original path no
longer exists:
+                    if (!verifyPathIntegrity(path, pathIdList, infoMap))
+                        return null;
+
+                    final IgfsFileInfo victimInfo = infoMap.get(victimId);
+
+                    if (!recursive && victimInfo.isDirectory() && !victimInfo.listing().isEmpty())
+                        // Throw exception if not empty and not recursive.
+                        throw new IgfsDirectoryNotEmptyException("Failed to remove directory
(directory is not " +
+                            "empty and recursive flag is not set).");
+
+                    IgfsFileInfo destInfo = infoMap.get(TRASH_ID);
 
-                    IgniteUuid resId = softDeleteNonTx(parentId, pathName, pathId);
+                    assert destInfo != null;
+
+                    final String srcFileName = path.name();
+
+                    final String destFileName = victimId.toString();
+
+                    assert destInfo.listing().get(destFileName) == null : "Failed to add
file name into the " +
+                        "destination directory (file already exists) [destName=" + destFileName
+ ']';
+
+                    IgfsFileInfo srcParentInfo = infoMap.get(pathIdList.get(pathIdList.size()
- 2));
+
+                    assert srcParentInfo != null;
+
+                    IgniteUuid srcParentId = srcParentInfo.id();
+                    assert srcParentId.equals(pathIdList.get(pathIdList.size() - 2));
+
+                    IgfsListingEntry srcEntry = srcParentInfo.listing().get(srcFileName);
+
+                    assert srcEntry != null : "Deletion victim not found in parent listing
[path=" + path +
+                        ", name=" + srcFileName + ", listing=" + srcParentInfo.listing()
+ ']';
+
+                    assert victimId.equals(srcEntry.fileId());
+
+                    id2InfoPrj.invoke(srcParentId, new UpdateListing(srcFileName, srcEntry,
true));
+
+                    // Add listing entry into the destination parent listing.
+                    id2InfoPrj.invoke(TRASH_ID, new UpdateListing(destFileName, srcEntry,
false));
+
+                    if (victimInfo.isFile())
+                        // Update a file info of the removed file with a file path,
+                        // which will be used by delete worker for event notifications.
+                        id2InfoPrj.invoke(victimId, new UpdatePath(path));
 
                     tx.commit();
 
                     delWorker.signal();
 
-                    return resId;
+                    return victimId;
                 }
                 finally {
                     tx.close();
@@ -1234,8 +1396,8 @@ public class IgfsMetaManager extends IgfsManager {
             }
         }
         else
-            throw new IllegalStateException("Failed to perform soft delete because Grid is
stopping [parentId=" +
-                parentId + ", pathName=" + pathName + ", pathId=" + pathId + ']');
+            throw new IllegalStateException("Failed to perform soft delete because Grid is
" +
+                "stopping [path=" + path + ']');
     }
 
     /**
@@ -1316,6 +1478,7 @@ public class IgfsMetaManager extends IgfsManager {
 
     /**
      * Remove listing entries of the given parent.
+     * This operation actually deletes directories from TRASH, is used solely by IgfsDeleteWorker.
      *
      * @param parentId Parent ID.
      * @param listing Listing entries.
@@ -1403,6 +1566,7 @@ public class IgfsMetaManager extends IgfsManager {
 
     /**
      * Remove entry from the metadata listing.
+     * Used solely by IgfsDeleteWorker.
      *
      * @param parentId Parent ID.
      * @param name Name.
@@ -2394,17 +2558,18 @@ public class IgfsMetaManager extends IgfsManager {
         if (busyLock.enterBusy()) {
             try {
                 SynchronizationTask<IgfsFileInfo> task = new SynchronizationTask<IgfsFileInfo>()
{
-                    @Override public IgfsFileInfo onSuccess(Map<IgfsPath, IgfsFileInfo>
infos)
-                        throws Exception {
+                    @Override public IgfsFileInfo onSuccess(Map<IgfsPath, IgfsFileInfo>
infos) throws Exception {
                         if (infos.get(path) == null)
                             return null;
 
                         fs.update(path, props);
 
-                        assert path.parent() == null || infos.get(path.parent()) != null;
+                        IgfsFileInfo parentInfo = infos.get(path.parent());
+
+                        assert path.parent() == null || parentInfo != null;
 
-                        return updatePropertiesNonTx(infos.get(path.parent()).id(), infos.get(path).id(),
path.name(),
-                            props);
+                        return updatePropertiesNonTx(parentInfo == null ? null : parentInfo.id(),
+                            infos.get(path).id(), path.name(), props);
                     }
 
                     @Override public IgfsFileInfo onFailure(@Nullable Exception err) throws
IgniteCheckedException {

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
index ebf91e0..fd4ec17 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
@@ -237,7 +237,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest
{
             U.sleep(200);
         }
 
-        igfs.delete(new IgfsPath("/"), true);
+        igfs.format();
 
         igfs.awaitDeletesAsync().get();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index cfa99ff..7e73859 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -101,10 +101,10 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     protected static final long BLOCK_SIZE = 32 * 1024 * 1024;
 
     /** Default repeat count. */
-    protected static final int REPEAT_CNT = 5;
+    protected static final int REPEAT_CNT = 5; // Diagnostic: ~100; Regression: 5
 
     /** Concurrent operations count. */
-    protected static final int OPS_CNT = 16;
+    protected static final int OPS_CNT = 16; // Diagnostic: ~160; Regression: 16
 
     /** Seed. */
     protected static final long SEED = System.currentTimeMillis();
@@ -252,6 +252,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         clear(igfs, igfsSecondary);
+
+        assert igfs.listFiles(new IgfsPath("/")).isEmpty();
     }
 
     /** {@inheritDoc} */
@@ -923,6 +925,23 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     }
 
     /**
+     * Tests that root directory properties persist afetr the #format() operation.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRootPropertiesPersistAfterFormat() throws Exception {
+        igfs.update(new IgfsPath("/"), Collections.singletonMap("foo", "moo"));
+
+        igfs.format();
+
+        IgfsFile file = igfs.info(new IgfsPath("/"));
+
+        Map<String,String> props = file.properties();
+
+        assertEquals("moo", props.get("foo"));
+    }
+
+    /**
      * Test regular file open.
      *
      * @throws Exception If failed.
@@ -1600,6 +1619,8 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @throws Exception If failed.
      */
     public void testConcurrentMkdirsDelete() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1541");
+
         for (int i = 0; i < REPEAT_CNT; i++) {
             final CyclicBarrier barrier = new CyclicBarrier(2);
 
@@ -1881,18 +1902,10 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     public void testDeadlocksRename() throws Exception {
         for (int i = 0; i < REPEAT_CNT; i++) {
             try {
-                info(">>>>>> Start deadlock test.");
-
                 checkDeadlocks(5, 2, 2, 2, OPS_CNT, 0, 0, 0, 0);
-
-                info(">>>>>> End deadlock test.");
             }
             finally {
-                info(">>>>>> Start cleanup.");
-
                 clear(igfs, igfsSecondary);
-
-                info(">>>>>> End cleanup.");
             }
         }
     }
@@ -1903,8 +1916,6 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @throws Exception If failed.
      */
     public void testDeadlocksDelete() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-1515");
-
         for (int i = 0; i < REPEAT_CNT; i++) {
             try {
                 checkDeadlocks(5, 2, 2, 2, 0, OPS_CNT, 0, 0, 0);
@@ -1948,6 +1959,42 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     }
 
     /**
+     * Ensure that deadlocks do not occur during concurrent delete & rename operations.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeadlocksDeleteRename() throws Exception {
+        for (int i = 0; i < REPEAT_CNT; i++) {
+            try {
+                checkDeadlocks(5, 2, 2, 2,
+                    OPS_CNT, OPS_CNT, 0, 0, 0);
+            }
+            finally {
+                clear(igfs, igfsSecondary);
+            }
+        }
+    }
+
+    /**
+     * Ensure that deadlocks do not occur during concurrent delete & rename operations.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeadlocksDeleteMkdirs() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1541");
+
+        for (int i = 0; i < REPEAT_CNT; i++) {
+            try {
+                checkDeadlocks(5, 2, 2, 2,
+                     0, OPS_CNT, 0, OPS_CNT, 0);
+            }
+            finally {
+                clear(igfs, igfsSecondary);
+            }
+        }
+    }
+
+    /**
      * Ensure that deadlocks do not occur during concurrent file creation operations.
      *
      * @throws Exception If failed.
@@ -1969,11 +2016,16 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @throws Exception If failed.
      */
     public void testDeadlocks() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-1515");
+        fail("https://issues.apache.org/jira/browse/IGNITE-1541");
 
         for (int i = 0; i < REPEAT_CNT; i++) {
             try {
-                checkDeadlocks(5, 2, 2, 2, OPS_CNT, OPS_CNT, OPS_CNT, OPS_CNT, OPS_CNT);
+                checkDeadlocks(5, 2, 2, 2,
+                    OPS_CNT, // rename
+                    OPS_CNT, // delete
+                    OPS_CNT, // update
+                    OPS_CNT, // mkdirs
+                    OPS_CNT); // create
             }
             finally {
                 clear(igfs, igfsSecondary);

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
index 206c9fe..4072636 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java
@@ -300,25 +300,19 @@ public class IgfsMetaManagerSelfTest extends IgfsCommonAbstractTest
{
         System.out.println("b: " + mgr.directoryListing(b.id()));
         System.out.println("f3: " + mgr.directoryListing(f3.id()));
 
-        //mgr.move(a.id(), "a", ROOT_ID, "a2", ROOT_ID);
         mgr.move(path("/a"), path("/a2"));
-        //mgr.move(b.id(), "b", a.id(), "b2", a.id());
         mgr.move(path("/a2/b"), path("/a2/b2"));
 
         assertNotNull(mgr.info(b.id()));
 
-        //mgr.move(f3.id(), "f3", b.id(), "f3-2", a.id());
         mgr.move(path("/a2/b2/f3"), path("/a2/b2/f3-2"));
 
         assertNotNull(mgr.info(b.id()));
 
-        //mgr.move(f3.id(), "f3-2", a.id(), "f3", b.id());
         mgr.move(path("/a2/b2/f3-2"), path("/a2/b2/f3"));
 
-        //mgr.move(b.id(), "b2", a.id(), "b", a.id());
         mgr.move(path("/a2/b2"), path("/a2/b"));
 
-        //mgr.move(a.id(), "a2", ROOT_ID, "a", ROOT_ID);
         mgr.move(path("/a2"), path("/a"));
 
         // Validate 'remove' operation.

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
index 8a2e5bf..fb1d6f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
@@ -348,7 +348,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
         assertEquals(0, m.filesOpenedForRead());
         assertEquals(0, m.filesOpenedForWrite());
 
-        fs.delete(new IgfsPath("/"), true);
+        fs.format();
 
         m = fs.metrics();
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
index cb134f8..9c4d832 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
@@ -389,15 +389,6 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
 
         assert paths.size() == 3 : "Unexpected paths: " + paths;
 
-        // Delete.
-        GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                igfs.delete(path("/"), false);
-
-                return null;
-            }
-        }, IgfsException.class, null);
-
         igfs.delete(path("/A1/B1/C1"), false);
         assertNull(igfs.info(path("/A1/B1/C1")));
 
@@ -416,19 +407,19 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
 
         assertEquals(Arrays.asList(path("/A"), path("/A1"), path("/A2")), sorted(igfs.listPaths(path("/"))));
 
-        GridTestUtils.assertThrowsInherited(log, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                igfs.delete(path("/"), false);
-
-                return null;
-            }
-        }, IgfsException.class, null);
-        assertEquals(Arrays.asList(path("/A"), path("/A1"), path("/A2")), sorted(igfs.listPaths(path("/"))));
-
+        // Delete root when it is not empty:
         igfs.delete(path("/"), true);
+        igfs.delete(path("/"), false);
+
+        igfs.delete(path("/A"), true);
+        igfs.delete(path("/A1"), true);
+        igfs.delete(path("/A2"), true);
         assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(path("/")));
 
+        // Delete root when it is empty:
         igfs.delete(path("/"), false);
+        igfs.delete(path("/"), true);
+
         assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(path("/")));
 
         for (Cache.Entry<Object, Object> e : metaCache)
@@ -608,7 +599,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
         assertEquals(text, read("/b"));
 
         // Cleanup.
-        igfs.delete(root, true);
+        igfs.format();
 
         assertEquals(Collections.<IgfsPath>emptyList(), igfs.listPaths(root));
     }

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
index 3bf70e2..ba8c164 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
@@ -28,7 +28,6 @@ import java.util.Map;
  * To be used solely in tests.
  */
 public interface UniversalFileSystemAdapter {
-
     /**
      * Gets name of the FS.
      * @return name of this file system.

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
index 3913cbd..fa5cbc5 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsUtils.java
@@ -20,7 +20,11 @@ package org.apache.ignite.internal.processors.hadoop.igfs;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathExistsException;
 import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
 import org.apache.ignite.IgniteCheckedException;
@@ -130,6 +134,38 @@ public class HadoopIgfsUtils {
     }
 
     /**
+     * Deletes all files from the given file system.
+     *
+     * @param fs The file system to clean up.
+     * @throws IOException On error.
+     */
+    public static void clear(FileSystem fs) throws IOException {
+        // Delete root contents:
+        FileStatus[] statuses = fs.listStatus(new Path("/"));
+
+        if (statuses != null) {
+            for (FileStatus stat: statuses)
+                fs.delete(stat.getPath(), true);
+        }
+    }
+
+    /**
+     * Deletes all files from the given file system.
+     *
+     * @param fs The file system to clean up.
+     * @throws IOException On error.
+     */
+    public static void clear(AbstractFileSystem fs) throws IOException {
+        // Delete root contents:
+        FileStatus[] statuses = fs.listStatus(new Path("/"));
+
+        if (statuses != null) {
+            for (FileStatus stat: statuses)
+                fs.delete(stat.getPath(), true);
+        }
+    }
+
+    /**
      * Constructor.
      */
     private HadoopIgfsUtils() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
index 03f0066..608bd25 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils;
 import org.apache.ignite.internal.processors.igfs.IgfsEx;
 import org.apache.ignite.internal.processors.igfs.UniversalFileSystemAdapter;
 
@@ -33,7 +34,6 @@ import org.apache.ignite.internal.processors.igfs.UniversalFileSystemAdapter;
  * Universal adapter wrapping {@link org.apache.hadoop.fs.FileSystem} instance.
  */
 public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFileSystemAdapter
{
-
     /** The wrapped filesystem. */
     private final FileSystem fileSys;
 
@@ -70,7 +70,7 @@ public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFile
 
     /** {@inheritDoc} */
     @Override public void format() throws IOException {
-        fileSys.delete(new Path("/"), true);
+        HadoopIgfsUtils.clear(fileSys);
     }
 
     /** {@inheritDoc} */

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
index 1235786..c938571 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
@@ -59,6 +59,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.FileSystemConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils;
 import org.apache.ignite.internal.processors.igfs.IgfsCommonAbstractTest;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.typedef.F;
@@ -352,7 +353,7 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         try {
-            fs.delete(new Path("/"), true);
+            HadoopIgfsUtils.clear(fs);
         }
         catch (Exception ignore) {
             // No-op.
@@ -627,7 +628,9 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA
 
         Path root = new Path(fsHome, "/");
 
-        assertTrue(fs.delete(root, true));
+        assertFalse(fs.delete(root, true));
+
+        assertTrue(fs.delete(new Path(fsHome, "/someDir1"), true));
 
         assertPathDoesNotExist(fs, someDir3);
         assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));

http://git-wip-us.apache.org/repos/asf/ignite/blob/54bb7d76/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 5ea841e..2626ebb 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -300,7 +300,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         try {
-            fs.delete(new Path("/"), true);
+            HadoopIgfsUtils.clear(fs);
         }
         catch (Exception ignore) {
             // No-op.
@@ -783,7 +783,8 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
 
         Path root = new Path(fsHome, "/");
 
-        assertTrue(fs.delete(root, true));
+        assertFalse(fs.delete(root, true));
+        assertTrue(fs.delete(new Path("/someDir1"), true));
 
         assertPathDoesNotExist(fs, someDir3);
         assertPathDoesNotExist(fs, new Path(fsHome, "/someDir1/someDir2"));


Mime
View raw message