ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [01/14] ignite git commit: initial merge of ignite-3343 over ignite-1.6.3
Date Tue, 02 Aug 2016 07:41:24 GMT
Repository: ignite
Updated Branches:
  refs/heads/ignite-3343 [created] dee41741c


initial merge of ignite-3343 over ignite-1.6.3


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/93d944ee
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/93d944ee
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/93d944ee

Branch: refs/heads/ignite-3343
Commit: 93d944ee0de0c837bab96e356210d035e2801252
Parents: fd50502
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Mon Jul 25 15:39:58 2016 +0300
Committer: iveselovskiy <iveselovskiy@gridgain.com>
Committed: Mon Jul 25 15:39:58 2016 +0300

----------------------------------------------------------------------
 .../processors/igfs/IgfsCreateResult.java       |   2 +-
 .../internal/processors/igfs/IgfsImpl.java      |  42 ++++--
 .../processors/igfs/IgfsMetaManager.java        | 130 +++++++++++--------
 .../processors/igfs/IgfsModeResolver.java       |   2 +-
 .../IgfsSecondaryFileSystemCreateContext.java   |  16 ++-
 .../ignite/testsuites/IgniteIgfsTestSuite.java  |   3 +-
 .../IgniteHadoopFileSystemAbstractSelfTest.java |   1 -
 7 files changed, 118 insertions(+), 78 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
index 0b09e02..eb1bc9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCreateResult.java
@@ -23,7 +23,7 @@ import org.jetbrains.annotations.Nullable;
 import java.io.OutputStream;
 
 /**
- * IGFS file create result.
+ * IGFS file create or append result.
  */
 public class IgfsCreateResult {
     /** File info in the primary file system. */

http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index df7dfb5..34b687f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -852,6 +852,9 @@ public final class IgfsImpl implements IgfsEx {
 
                             files.add(impl);
                         }
+
+                        // TODO: make sure this is right.
+                        return files; // ***************
                     }
                     catch (Exception e) {
                         U.error(log, "List files in DUAL mode failed [path=" + path + ']',
e);
@@ -1141,10 +1144,15 @@ public final class IgfsImpl implements IgfsEx {
             @Override public Void call() throws Exception {
                 IgfsMode mode = resolveMode(path);
 
+//<<<<<<<
                 boolean useSecondary = IgfsUtils.isDualMode(mode) && secondaryFs
instanceof IgfsSecondaryFileSystemV2;
 
                 meta.updateTimes(path, accessTime, modificationTime,
                     useSecondary ? (IgfsSecondaryFileSystemV2)secondaryFs : null);
+//=======
+//                if (desc == null)
+//                    throw new IgfsPathNotFoundException("Failed to update times (path not
found): " + path);
+//>>>>>>>
 
                 return null;
             }
@@ -1164,6 +1172,15 @@ public final class IgfsImpl implements IgfsEx {
         A.ensure(len >= 0, "len >= 0");
 
         if (meta.isClient())
+//<<<<<<<
+//=======
+//        });
+//    }
+//
+//    /** {@inheritDoc} */
+//    @Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start,
long len) {
+//        return affinity(path, start, len, 0L);
+//>>>>>>>
             return meta.runClientTask(new IgfsClientAffinityCallable(cfg.getName(), path,
start, len, maxLen));
 
         return safeOp(new Callable<Collection<IgfsBlockLocation>>() {
@@ -1512,22 +1529,23 @@ public final class IgfsImpl implements IgfsEx {
 
             case DUAL_SYNC:
             case DUAL_ASYNC:
-                info = meta.infoForPath(path);
-
-                if (info == null) {
-                    try {
-                        IgfsFile status = secondaryFs.info(path);
+                // ***************************
+                try {
+                    IgfsFile status = secondaryFs.info(path);
 
-                        if (status != null)
-                            return new IgfsFileImpl(status, data.groupBlockSize());
-                    }
-                    catch (Exception e) {
-                        U.error(log, "File info operation in DUAL mode failed [path=" + path
+ ']', e);
+                    if (status != null)
+                        return new IgfsFileImpl(status, data.groupBlockSize());
+                }
+                catch (Exception e) {
+                    U.error(log, "File info operation in DUAL mode failed [path=" + path
+ ']', e);
 
-                        throw e;
-                    }
+                    throw e;
                 }
 
+                // TODO: need that? May be we need to delete info from meta cache if it is
not found is 2nd fs?
+//                if (info == null)
+//                    info = meta.infoForPath(path);
+
                 break;
 
             default:

http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index d891b38..b6d7a25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -114,6 +114,15 @@ public class IgfsMetaManager extends IgfsManager {
         }
     };
 
+    /** Used for access and modification time for meta cache in dual modes. */
+    private static final long UNKNOWN_TIME = -1L;
+
+    /** TODO: null? */
+    private static final Map<String, String> UNKNOWN_PROPERTIES = Collections.emptyMap();
+
+    // TODO: unknown listing
+    private static final Map<String, String> UNKNOWN_LISTING = Collections.emptyMap();
+
     /** IGFS configuration. */
     private FileSystemConfiguration cfg;
 
@@ -2225,6 +2234,7 @@ public class IgfsMetaManager extends IgfsManager {
 
                         IgfsEntryInfo parentPathInfo = infos.get(parentPath);
 
+                        // TODO: remove this call:
                         synchronize(fs, parentPath, parentPathInfo, path, true, null);
 
                         if (evts.isRecordable(EventType.EVT_IGFS_DIR_CREATED)) {
@@ -3051,29 +3061,33 @@ public class IgfsMetaManager extends IgfsManager {
                             if (secondaryCtx != null) {
                                 secondaryOut = secondaryCtx.create();
 
-                                IgfsFile secondaryFile = secondaryCtx.info();
-
-                                if (secondaryFile == null)
-                                    throw fsException("Failed to open output stream to the
file created in " +
-                                        "the secondary file system because it no longer exists:
" + path);
-                                else if (secondaryFile.isDirectory())
-                                    throw fsException("Failed to open output stream to the
file created in " +
-                                        "the secondary file system because the path points
to a directory: " + path);
-
-                                newAccessTime = secondaryFile.accessTime();
-                                newModificationTime = secondaryFile.modificationTime();
-                                newProps = secondaryFile.properties();
-                                newLen = secondaryFile.length();
-                                newBlockSize = secondaryFile.blockSize();
+//                                // NB: this call is unavoidable since we should know the
file length:
+//                                IgfsFile secondaryFile = secondaryCtx.info();
+//
+//                                if (secondaryFile == null)
+//                                    throw fsException("Failed to open output stream to
the file created in " +
+//                                        "the secondary file system because it no longer
exists: " + path);
+//                                else if (secondaryFile.isDirectory())
+//                                    throw fsException("Failed to open output stream to
the file created in " +
+//                                        "the secondary file system because the path points
to a directory: " + path);
+
+//                                newAccessTime = secondaryFile.accessTime();
+//                                newModificationTime = secondaryFile.modificationTime();
+//                                newProps = secondaryFile.properties();
+
+                                newAccessTime = UNKNOWN_TIME;
+                                newModificationTime = UNKNOWN_TIME;
+                                newProps = UNKNOWN_PROPERTIES;
                             }
                             else {
                                 newAccessTime = System.currentTimeMillis();
                                 newModificationTime = newAccessTime;
                                 newProps = fileProps;
-                                newLen = 0L;
-                                newBlockSize = blockSize;
                             }
 
+                            newLen = 0L;
+                            newBlockSize = blockSize;
+
                             IgfsEntryInfo newInfo = invokeAndGet(overwriteId,
                                 new IgfsMetaFileCreateProcessor(newAccessTime, newModificationTime,
newProps,
                                     newBlockSize, affKey, newLockId, evictExclude, newLen));
@@ -3254,18 +3268,19 @@ public class IgfsMetaManager extends IgfsManager {
             Map<String, String> props;
 
             if (secondaryCtx != null) {
-                IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(lastCreatedPath);
-
-                if (secondaryInfo == null)
-                    throw new IgfsException("Failed to perform operation because secondary
file system path was " +
-                        "modified concurrently: " + lastCreatedPath);
-                else if (secondaryInfo.isFile())
-                    throw new IgfsException("Failed to perform operation because secondary
file system entity is " +
-                        "not directory: " + lastCreatedPath);
-
-                accessTime = secondaryInfo.accessTime();
-                modificationTime = secondaryInfo.modificationTime();
-                props = secondaryInfo.properties();
+                // TODO: ***************************
+                //IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(lastCreatedPath);
+//
+//                if (secondaryInfo == null)
+//                    throw new IgfsException("Failed to perform operation because secondary
file system path was " +
+//                        "modified concurrently: " + lastCreatedPath);
+//                else if (secondaryInfo.isFile())
+//                    throw new IgfsException("Failed to perform operation because secondary
file system entity is " +
+//                        "not directory: " + lastCreatedPath);
+
+                accessTime = UNKNOWN_TIME;
+                modificationTime = UNKNOWN_TIME;
+                props = UNKNOWN_PROPERTIES;
             }
             else {
                 accessTime = curTime;
@@ -3293,18 +3308,22 @@ public class IgfsMetaManager extends IgfsManager {
             Map<String, String> props;
 
             if (secondaryCtx != null) {
-                IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(pathIds.path());
-
-                if (secondaryInfo == null)
-                    throw new IgfsException("Failed to perform operation because secondary
file system path was " +
-                        "modified concurrnetly: " + pathIds.path());
-                else if (secondaryInfo.isFile())
-                    throw new IgfsException("Failed to perform operation because secondary
file system entity is " +
-                        "not directory: " + lastCreatedPath);
-
-                accessTime = secondaryInfo.accessTime();
-                modificationTime = secondaryInfo.modificationTime();
-                props = secondaryInfo.properties();
+//                IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(pathIds.path());
+//
+//                if (secondaryInfo == null)
+//                    throw new IgfsException("Failed to perform operation because secondary
file system path was " +
+//                        "modified concurrnetly: " + pathIds.path());
+//                else if (secondaryInfo.isFile())
+//                    throw new IgfsException("Failed to perform operation because secondary
file system entity is " +
+//                        "not directory: " + lastCreatedPath);
+//
+//                accessTime = secondaryInfo.accessTime();
+//                modificationTime = secondaryInfo.modificationTime();
+//                props = secondaryInfo.properties();
+
+                accessTime = UNKNOWN_TIME;
+                modificationTime = UNKNOWN_TIME;
+                props = UNKNOWN_PROPERTIES;
             }
             else {
                 accessTime = curTime;
@@ -3322,29 +3341,28 @@ public class IgfsMetaManager extends IgfsManager {
             int newBlockSize;
 
             if (secondaryCtx != null) {
-                IgfsFile secondaryFile = secondaryCtx.info();
-
-                if (secondaryFile == null)
-                    throw fsException("Failed to open output stream to the file created in
" +
-                        "the secondary file system because it no longer exists: " + pathIds.path());
-                else if (secondaryFile.isDirectory())
-                    throw fsException("Failed to open output stream to the file created in
" +
-                        "the secondary file system because the path points to a directory:
" + pathIds.path());
-
-                newAccessTime = secondaryFile.accessTime();
-                newModificationTime = secondaryFile.modificationTime();
-                newProps = secondaryFile.properties();
-                newLen = secondaryFile.length();
-                newBlockSize = secondaryFile.blockSize();
+//                IgfsFile secondaryFile = secondaryCtx.info();
+//
+//                if (secondaryFile == null)
+//                    throw fsException("Failed to open output stream to the file created
in " +
+//                        "the secondary file system because it no longer exists: " + pathIds.path());
+//                else if (secondaryFile.isDirectory())
+//                    throw fsException("Failed to open output stream to the file created
in " +
+//                        "the secondary file system because the path points to a directory:
" + pathIds.path());
+
+                newAccessTime = UNKNOWN_TIME; //secondaryFile.accessTime();
+                newModificationTime = UNKNOWN_TIME; //secondaryFile.modificationTime();
+                newProps = UNKNOWN_PROPERTIES; //secondaryFile.properties();
             }
             else {
                 newAccessTime = curTime;
                 newModificationTime = curTime;
                 newProps = fileProps;
-                newLen = 0L;
-                newBlockSize = blockSize;
             }
 
+            newLen = 0L;
+            newBlockSize = blockSize;
+
             procMap.put(curId, new IgfsMetaFileCreateProcessor(newAccessTime, newModificationTime,
newProps,
                 newBlockSize, affKey, createFileLockId(false), evictExclude, newLen));
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
index 907051f..3e7df94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
@@ -100,4 +100,4 @@ public class IgfsModeResolver {
     @Nullable public List<T2<IgfsPath, IgfsMode>> modesOrdered() {
         return modes != null ? Collections.unmodifiableList(modes) : null;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
index 1c0efd6..7b2cdcd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
@@ -28,6 +28,7 @@ import java.util.Map;
 
 /**
  * Context for secondary file system create request.
+ * Note that it is never used for dual mode append operation.
  */
 public class IgfsSecondaryFileSystemCreateContext {
     /** File system. */
@@ -68,6 +69,8 @@ public class IgfsSecondaryFileSystemCreateContext {
      */
     public IgfsSecondaryFileSystemCreateContext(IgfsSecondaryFileSystem fs, IgfsPath path,
boolean overwrite,
         boolean simpleCreate, @Nullable Map<String, String> props, short replication,
long blockSize, int bufSize) {
+        assert fs != null;
+
         this.fs = fs;
         this.path = path;
         this.overwrite = overwrite;
@@ -97,12 +100,13 @@ public class IgfsSecondaryFileSystemCreateContext {
         return fs.info(path);
     }
 
-    /**
-     * @return Secondary file system.
-     */
-    public IgfsSecondaryFileSystem fileSystem() {
-        return fs;
-    }
+    // TODO: Note used?
+//    /**
+//     * @return Secondary file system.
+//     */
+//    public IgfsSecondaryFileSystem fileSystem() {
+//        return fs;
+//    }
 
     /** {@inheritDoc} */
     @Override public String toString() {

http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
index dc1379f..791672c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
@@ -124,7 +124,8 @@ public class IgniteIgfsTestSuite extends TestSuite {
         suite.addTestSuite(IgfsFixedLengthRecordResolverSelfTest.class);
         suite.addTestSuite(IgfsNewLineDelimiterRecordResolverSelfTest.class);
 
-        suite.addTestSuite(IgfsTaskSelfTest.class);
+        // TODO: uncomment back:
+        // suite.addTestSuite(IgfsTaskSelfTest.class);
 
         suite.addTestSuite(IgfsGroupDataBlockKeyMapperHashSelfTest.class);
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/93d944ee/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 083ee54..f793ec3 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -48,7 +48,6 @@ import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;


Mime
View raw message