ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject incubator-ignite git commit: [IGNITE-406]: part 1: adapted core tests for Hadoop underlying stack.
Date Wed, 11 Mar 2015 19:48:42 GMT
Repository: incubator-ignite
Updated Branches:
  refs/heads/ignite-406 [created] 50a789f56


[IGNITE-406]: part 1: adapted core tests for Hadoop underlying stack.


Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/50a789f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/50a789f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/50a789f5

Branch: refs/heads/ignite-406
Commit: 50a789f560ea978c7f9aafd49d2a40dea3894286
Parents: 9d78a08
Author: iveselovskiy <iveselovskiy@gridgain.com>
Authored: Wed Mar 11 22:47:57 2015 +0300
Committer: iveselovskiy <iveselovskiy@gridgain.com>
Committed: Wed Mar 11 22:47:57 2015 +0300

----------------------------------------------------------------------
 .../processors/igfs/IgfsAbstractSelfTest.java   | 229 +++++++++++++++++--
 .../igfs/IgfsDualAbstractSelfTest.java          |  69 +++---
 .../igfs/IgfsExUniversalFileSystemAdapter.java  |  71 ++++++
 .../igfs/UniversalFileSystemAdapter.java        |  64 ++++++
 .../fs/IgniteHadoopIgfsSecondaryFileSystem.java |   8 +
 .../ignite/igfs/Hadoop10DualAbstractTest.java   |  59 +++++
 .../igfs/Hadoop1OverIgfsDualAsyncTest.java      |  12 +
 .../igfs/Hadoop1OverIgfsDualSyncTest.java       |  22 ++
 ...oopFileSystemUniversalFileSystemAdapter.java |  75 ++++++
 ...oopSecondaryFileSystemConfigurationTest.java |  14 +-
 10 files changed, 558 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 9dde005..58a5e61 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -110,13 +110,16 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     protected static final IgfsPath FILE_NEW = new IgfsPath(SUBDIR_NEW, "fileNew");
 
     /** Default data chunk (128 bytes). */
-    protected static byte[] chunk;
+    protected static final byte[] chunk = createChunk(128);
 
     /** Primary IGFS. */
     protected static IgfsImpl igfs;
 
-    /** Secondary IGFS. */
-    protected static IgfsImpl igfsSecondary;
+    /** */
+    protected static IgfsSecondaryFileSystem igfsSecondaryFileSystem;
+
+    /** Secondary file system "backdoor". */
+    protected static UniversalFileSystemAdapter igfsSecondary;
 
     /** IGFS mode. */
     protected final IgfsMode mode;
@@ -145,29 +148,45 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         dual = mode != PRIMARY;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        chunk = new byte[128];
+    private static byte[] createChunk(int length) {
+        byte[] chunk = new byte[length];
 
         for (int i = 0; i < chunk.length; i++)
             chunk[i] = (byte)i;
 
-        Ignite igniteSecondary = startGridWithIgfs("ignite-secondary", "igfs-secondary",
PRIMARY, null, SECONDARY_REST_CFG);
+        return chunk;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        U.debug("============================= beforeTestsStarted()");
 
-        igfsSecondary = (IgfsImpl) igniteSecondary.fileSystem("igfs-secondary");
+        igfsSecondaryFileSystem = createSecondaryFileSystemStack();
 
-        Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondary.asSecondary(),
PRIMARY_REST_CFG);
+        Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondaryFileSystem,
PRIMARY_REST_CFG);
 
         igfs = (IgfsImpl) ignite.fileSystem("igfs");
     }
 
+    protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception {
+        Ignite igniteSecondary = startGridWithIgfs("ignite-secondary", "igfs-secondary",
PRIMARY, null, SECONDARY_REST_CFG);
+
+        IgfsEx secondaryIgfsImpl = (IgfsEx) igniteSecondary.fileSystem("igfs-secondary");
+
+        igfsSecondary = new IgfsExUniversalFileSystemAdapter(secondaryIgfsImpl);
+
+        return secondaryIgfsImpl.asSecondary();
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
+        U.debug("========== afterTest()");
         clear(igfs, igfsSecondary);
     }
 
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
+        U.debug("============================= afterTestsStopped()");
         G.stopAll(true);
     }
 
@@ -634,7 +653,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         checkExist(igfs, igfsSecondary, SUBSUBDIR);
 
         if (dual)
-            assertEquals(props, igfsSecondary.info(SUBSUBDIR).properties());
+            assertEquals(props, igfsSecondary.properties(SUBSUBDIR.toString()));
 
         // We check only permission because IGFS client adds username and group name explicitly.
         assertEquals(props.get(PROP_PERMISSION), igfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION));
@@ -653,7 +672,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         checkExist(igfs, igfsSecondary, DIR);
 
         if (dual)
-            assertEquals(props, igfsSecondary.info(DIR).properties());
+            assertEquals(props, igfsSecondary.properties(DIR.toString()));
 
         // We check only permission because IGFS client adds username and group name explicitly.
         assertEquals(props.get(PROP_PERMISSION), igfs.info(DIR).properties().get(PROP_PERMISSION));
@@ -718,7 +737,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         igfs.update(FILE, props);
 
         if (dual)
-            assertEquals(props, igfsSecondary.info(FILE).properties());
+            assertEquals(props, igfsSecondary.properties(FILE.toString()));
 
         assertEquals(props, igfs.info(FILE).properties());
     }
@@ -736,7 +755,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         igfs.update(DIR, props);
 
         if (dual)
-            assertEquals(props, igfsSecondary.info(DIR).properties());
+            assertEquals(props, igfsSecondary.properties(DIR.toString()));
 
         assertEquals(props, igfs.info(DIR).properties());
     }
@@ -853,7 +872,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @throws Exception If failed.
      */
     public void testOpenDoesNotExist() throws Exception {
-        igfsSecondary.delete(FILE, false);
+        igfsSecondary.delete(FILE.toString(), false);
 
         GridTestUtils.assertThrows(log(), new Callable<Object>() {
             @Override public Object call() throws Exception {
@@ -877,7 +896,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @throws Exception If failed.
      */
     public void testCreate() throws Exception {
-        create(igfs.asSecondary(), paths(DIR, SUBDIR), null);
+        create(igfs, paths(DIR, SUBDIR), null);
 
         createFile(igfs.asSecondary(), FILE, true, chunk);
 
@@ -2106,7 +2125,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         for (int i = 0; i < lvlCnt; i++) {
             int lvl = i + 1;
 
-            IgfsImpl targetIgfs = dual ? lvl <= primaryLvlCnt ? igfs : igfsSecondary :
igfs;
+            // TODO: (IgfsImpl)igfsSecondary.getAdapter(IgfsEx.class)) will return null in
non-IgfsEx case.
+            IgfsImpl targetIgfs = dual ?
+                (lvl <= primaryLvlCnt ? igfs : (IgfsImpl)igfsSecondary.getAdapter(IgfsEx.class))
: igfs;
 
             IgfsPath[] dirs = dirPaths.get(lvl).toArray(new IgfsPath[dirPaths.get(lvl).size()]);
             IgfsPath[] files = filePaths.get(lvl).toArray(new IgfsPath[filePaths.get(lvl).size()]);
@@ -2130,6 +2151,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @throws Exception If failed.
      */
     public static void create(IgfsImpl igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[]
files) throws Exception {
+        // TODO: why .asSecondary() is used there??
         create(igfs.asSecondary(), dirs, files);
     }
 
@@ -2157,6 +2179,20 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         }
     }
 
+    public void create(UniversalFileSystemAdapter uni, @Nullable IgfsPath[] dirs, @Nullable
IgfsPath[] files) throws Exception {
+        if (dirs != null) {
+            for (IgfsPath dir : dirs)
+                uni.mkdirs(dir.toString());
+        }
+
+        if (files != null) {
+            for (IgfsPath file : files)
+                try (OutputStream os = uni.openOutputStream(file.toString(), false)) {
+                    // noop
+                }
+        }
+    }
+
     /**
      * Create the file in the given IGFS and write provided data chunks to it.
      *
@@ -2185,6 +2221,32 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     /**
      * Create the file in the given IGFS and write provided data chunks to it.
      *
+     * @param file File.
+     * @param overwrite Overwrite flag.
+     * @param chunks Data chunks.
+     * @throws IOException In case of IO exception.
+     */
+    protected static void createFile(UniversalFileSystemAdapter uni, IgfsPath file, boolean
overwrite, @Nullable byte[]... chunks)
+        throws IOException {
+        OutputStream os = null;
+
+        try {
+            os = uni.openOutputStream(file.toString(), false);
+
+            writeFileChunks(os, chunks);
+        }
+        finally {
+            U.closeQuiet(os);
+
+            IgfsEx igfsEx = uni.getAdapter(IgfsEx.class);
+            if (igfsEx != null)
+                awaitFileClose(igfsEx.asSecondary(), file);
+        }
+    }
+
+    /**
+     * Create the file in the given IGFS and write provided data chunks to it.
+     *
      * @param igfs IGFS.
      * @param file File.
      * @param overwrite Overwrite flag.
@@ -2269,7 +2331,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @param paths Paths.
      * @throws Exception If failed.
      */
-    protected void checkExist(IgfsImpl igfs, IgfsImpl igfsSecondary, IgfsPath... paths) throws
Exception {
+    protected void checkExist(IgfsImpl igfs, UniversalFileSystemAdapter igfsSecondary, IgfsPath...
paths) throws Exception {
         checkExist(igfs, paths);
 
         if (dual)
@@ -2292,6 +2354,27 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     }
 
     /**
+     * Ensure that the given paths exist in the given IGFS.
+     *
+     * @param uni filesystem.
+     * @param paths Paths.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void checkExist(UniversalFileSystemAdapter uni, IgfsPath... paths) throws IgniteCheckedException
{
+        for (IgfsPath path : paths) {
+            // TODO: what does that mean?
+//            assert igfs.context().meta().fileId(path) != null : "Path doesn't exist [igfs="
+ igfs.name() +
+//                ", path=" + path + ']';
+
+            try {
+                assert uni.exists(path.toString()) : "Path doesn't exist [igfs=" + uni.name()
+ ", path=" + path + ']';
+            } catch (IOException ioe) {
+                throw new IgniteCheckedException(ioe);
+            }
+        }
+    }
+
+    /**
      * Ensure that the given paths don't exist in the given IGFSs.
      *
      * @param igfs First IGFS.
@@ -2299,7 +2382,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @param paths Paths.
      * @throws Exception If failed.
      */
-    protected void checkNotExist(IgfsImpl igfs, IgfsImpl igfsSecondary, IgfsPath... paths)
+    protected void checkNotExist(IgfsImpl igfs, UniversalFileSystemAdapter igfsSecondary,
IgfsPath... paths)
         throws Exception {
         checkNotExist(igfs, paths);
 
@@ -2323,6 +2406,42 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     }
 
     /**
+     * Ensure that the given paths don't exist in the given IGFS.
+     *
+     * @param uni secondary FS.
+     * @param paths Paths.
+     * @throws Exception If failed.
+     */
+    protected void checkNotExist(UniversalFileSystemAdapter uni, IgfsPath... paths) throws
Exception {
+        for (IgfsPath path : paths) {
+//            assert igfs.context().meta().fileId(path) == null : "Path exists [igfs=" +
igfs.name() + ", path=" +
+//                path + ']';
+            assert !uni.exists(path.toString()) : "Path exists [igfs=" + uni.name() + ",
path=" + path + ']';
+        }
+    }
+
+//    /**
+//     * Ensure that the given file exists in the given IGFSs and that it has exactly the
same content as provided in the
+//     * "data" parameter.
+//     *
+//     * @param igfs First IGFS.
+//     * @param igfsSecondary Second IGFS.
+//     * @param file File.
+//     * @param chunks Expected data.
+//     * @throws Exception If failed.
+//     */
+//    protected void checkFile(IgfsImpl igfs, IgfsImpl igfsSecondary, IgfsPath file,
+//        @Nullable byte[]... chunks) throws Exception {
+//        checkExist(igfs, file);
+//        checkFileContent(igfs, file, chunks);
+//
+//        if (dual) {
+//            checkExist(igfsSecondary, file);
+//            checkFileContent(igfsSecondary, file, chunks);
+//        }
+//    }
+
+    /**
      * Ensure that the given file exists in the given IGFSs and that it has exactly the same
content as provided in the
      * "data" parameter.
      *
@@ -2332,14 +2451,14 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @param chunks Expected data.
      * @throws Exception If failed.
      */
-    protected void checkFile(IgfsImpl igfs, IgfsImpl igfsSecondary, IgfsPath file,
+    protected void checkFile(IgfsImpl igfs, UniversalFileSystemAdapter igfsSecondary, IgfsPath
file,
         @Nullable byte[]... chunks) throws Exception {
         checkExist(igfs, file);
         checkFileContent(igfs, file, chunks);
 
         if (dual) {
             checkExist(igfsSecondary, file);
-            checkFileContent(igfsSecondary, file, chunks);
+            checkFileContent(igfsSecondary, file.toString(), chunks);
         }
     }
 
@@ -2382,6 +2501,46 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
     }
 
     /**
+     * Ensure that the given file has exactly the same content as provided in the "data"
parameter.
+     *
+     * @param uni FS.
+     * @param path File.
+     * @param chunks Expected data.
+     * @throws IOException In case of IO exception.
+     * @throws IgniteCheckedException In case of Grid exception.
+     */
+    protected void checkFileContent(UniversalFileSystemAdapter uni, String path, @Nullable
byte[]... chunks)
+        throws IOException, IgniteCheckedException {
+        if (chunks != null && chunks.length > 0) {
+            InputStream is = null;
+
+            try {
+                is = uni.openInputStream(path);
+
+                int chunkIdx = 0;
+
+                int read;
+                for (byte[] chunk: chunks) {
+                    byte[] buf = new byte[chunk.length];
+
+                    read = is.read(buf);
+
+                    assert read == chunk.length : "Chunk #" + chunkIdx + " was not read fully.";
+                    assert Arrays.equals(chunk, buf) : "Bad chunk [igfs=" + uni.name() +
", chunkIdx=" + chunkIdx +
+                        ", expected=" + Arrays.toString(chunk) + ", actual=" + Arrays.toString(buf)
+ ']';
+
+                    chunkIdx++;
+                }
+
+                is.close();
+            }
+            finally {
+                U.closeQuiet(is);
+            }
+        }
+    }
+
+    /**
      * Create map with properties.
      *
      * @param username User name.
@@ -2422,7 +2581,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
      * @param igfsSecondary Second IGFS.
      * @throws Exception If failed.
      */
-    protected void clear(IgniteFileSystem igfs, IgniteFileSystem igfsSecondary) throws Exception
{
+    protected void clear(IgniteFileSystem igfs, UniversalFileSystemAdapter igfsSecondary)
throws Exception {
         clear(igfs);
 
         if (dual)
@@ -2453,4 +2612,32 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest
{
         // Clear igfs.
         igfs.format();
     }
+
+    /**
+     * Clear particular {@link UniversalFileSystemAdapter}.
+     *
+     * @param uni IGFS.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    public static void clear(UniversalFileSystemAdapter uni) throws Exception {
+        IgfsEx igfsEx = uni.getAdapter(IgfsEx.class);
+        if (igfsEx != null) {
+            Field workerMapFld = IgfsImpl.class.getDeclaredField("workerMap");
+
+            workerMapFld.setAccessible(true);
+
+            // Wait for all workers to finish.
+            Map<IgfsPath, IgfsFileWorker> workerMap = (Map<IgfsPath, IgfsFileWorker>)workerMapFld.get(igfsEx);
+
+            for (Map.Entry<IgfsPath, IgfsFileWorker> entry : workerMap.entrySet())
{
+                entry.getValue().cancel();
+
+                U.join(entry.getValue());
+            }
+
+        }
+        // Clear igfs.
+        uni.format();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
index 1df79cc..9f2064c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
@@ -962,7 +962,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         checkExist(igfs, SUBDIR);
         checkExist(igfs, igfsSecondary, SUBSUBDIR);
 
-        assertEquals(props, igfsSecondary.info(SUBSUBDIR).properties());
+        assertEquals(props, igfsSecondary.properties(SUBSUBDIR.toString()));
 
         // We check only permission because IGFS client adds username and group name explicitly.
         assertEquals(props.get(PROP_PERMISSION), igfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION));
@@ -986,7 +986,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         checkExist(igfs, SUBDIR);
         checkExist(igfs, igfsSecondary, SUBSUBDIR);
 
-        assertEquals(props, igfsSecondary.info(SUBSUBDIR).properties());
+        assertEquals(props, igfsSecondary.properties(SUBSUBDIR.toString()));
 
         // We check only permission because IGFS client adds username and group name explicitly.
         assertEquals(props.get(PROP_PERMISSION), igfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION));
@@ -1049,7 +1049,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         create(igfs, paths(DIR), null);
 
         // Set different properties to the sub-directory.
-        igfsSecondary.update(SUBDIR, propsSubDir);
+        igfsSecondaryFileSystem.update(SUBDIR, propsSubDir);
 
         igfs.update(FILE, propsFile);
 
@@ -1057,10 +1057,10 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         checkExist(igfs, SUBDIR, FILE);
 
         // Ensure properties propagation.
-        assertEquals(propsSubDir, igfsSecondary.info(SUBDIR).properties());
+        assertEquals(propsSubDir, igfsSecondary.properties(SUBDIR.toString()));
         assertEquals(propsSubDir, igfs.info(SUBDIR).properties());
 
-        assertEquals(propsFile, igfsSecondary.info(FILE).properties());
+        assertEquals(propsFile, igfsSecondary.properties(FILE.toString()));
         assertEquals(propsFile, igfs.info(FILE).properties());
     }
 
@@ -1077,7 +1077,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         create(igfs, null, null);
 
         // Set different properties to the sub-directory.
-        igfsSecondary.update(SUBDIR, propsSubDir);
+        igfsSecondaryFileSystem.update(SUBDIR, propsSubDir);
 
         igfs.update(FILE, propsFile);
 
@@ -1085,10 +1085,10 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         checkExist(igfs, DIR, SUBDIR, FILE);
 
         // Ensure properties propagation.
-        assertEquals(propsSubDir, igfsSecondary.info(SUBDIR).properties());
+        assertEquals(propsSubDir, igfsSecondary.properties(SUBDIR.toString()));
         assertEquals(propsSubDir, igfs.info(SUBDIR).properties());
 
-        assertEquals(propsFile, igfsSecondary.info(FILE).properties());
+        assertEquals(propsFile, igfsSecondary.properties(FILE.toString()));
         assertEquals(propsFile, igfs.info(FILE).properties());
     }
 
@@ -1107,7 +1107,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
 
         checkExist(igfs, DIR);
 
-        assertEquals(props, igfsSecondary.info(DIR).properties());
+        assertEquals(props, igfsSecondary.properties(DIR.toString()));
         assertEquals(props, igfs.info(DIR).properties());
     }
 
@@ -1120,7 +1120,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         create(igfsSecondary, paths(DIR, SUBDIR), null);
         create(igfs, null, null);
 
-        createFile(igfsSecondary.asSecondary(), FILE, true, chunk);
+        createFile(igfsSecondary, FILE, true, chunk);
 
         checkFileContent(igfs, FILE, chunk);
     }
@@ -1136,19 +1136,17 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         // Write enough data to the secondary file system.
         final int blockSize = IGFS_BLOCK_SIZE;
 
-        IgfsOutputStream out = igfsSecondary.append(FILE, false);
-
         int totalWritten = 0;
+        try (OutputStream out = igfsSecondary.openOutputStream(FILE.toString(), false)) {
 
-        while (totalWritten < blockSize * 2 + chunk.length) {
-            out.write(chunk);
+            while (totalWritten < blockSize * 2 + chunk.length) {
+                out.write(chunk);
 
-            totalWritten += chunk.length;
+                totalWritten += chunk.length;
+            }
         }
 
-        out.close();
-
-        awaitFileClose(igfsSecondary.asSecondary(), FILE);
+        awaitFileClose(igfsSecondaryFileSystem, FILE);
 
         // Read the first block.
         int totalRead = 0;
@@ -1179,7 +1177,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         U.sleep(300);
 
         // Remove the file from the secondary file system.
-        igfsSecondary.delete(FILE, false);
+        igfsSecondary.delete(FILE.toString(), false);
 
         // Let's wait for file will be deleted.
         U.sleep(300);
@@ -1215,19 +1213,16 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         // Write enough data to the secondary file system.
         final int blockSize = igfs.info(FILE).blockSize();
 
-        IgfsOutputStream out = igfsSecondary.append(FILE, false);
-
         int totalWritten = 0;
+        try (OutputStream out = igfsSecondary.openOutputStream(FILE.toString(), false)) {
+            while (totalWritten < blockSize * 2 + chunk.length) {
+                out.write(chunk);
 
-        while (totalWritten < blockSize * 2 + chunk.length) {
-            out.write(chunk);
-
-            totalWritten += chunk.length;
+                totalWritten += chunk.length;
+            }
         }
 
-        out.close();
-
-        awaitFileClose(igfsSecondary.asSecondary(), FILE);
+        awaitFileClose(igfsSecondaryFileSystem, FILE);
 
         // Read the first two blocks.
         int totalRead = 0;
@@ -1260,7 +1255,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         }
 
         // Remove the file from the secondary file system.
-        igfsSecondary.delete(FILE, false);
+        igfsSecondary.delete(FILE.toString(), false);
 
         // Let's wait for file will be deleted.
         U.sleep(300);
@@ -1290,7 +1285,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         create(igfsSecondary, paths(DIR, SUBDIR), null);
         create(igfs, paths(DIR), null);
 
-        igfsSecondary.update(SUBDIR, props);
+        igfsSecondaryFileSystem.update(SUBDIR, props);
 
         createFile(igfs.asSecondary(), FILE, true, chunk);
 
@@ -1314,8 +1309,8 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         create(igfsSecondary, paths(DIR, SUBDIR), null);
         create(igfs, null, null);
 
-        igfsSecondary.update(DIR, propsDir);
-        igfsSecondary.update(SUBDIR, propsSubDir);
+        igfsSecondaryFileSystem.update(DIR, propsDir);
+        igfsSecondaryFileSystem.update(SUBDIR, propsSubDir);
 
         createFile(igfs.asSecondary(), FILE, true, chunk);
 
@@ -1338,9 +1333,9 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         create(igfsSecondary, paths(DIR, SUBDIR), null);
         create(igfs, paths(DIR), null);
 
-        igfsSecondary.update(SUBDIR, props);
+        igfsSecondaryFileSystem.update(SUBDIR, props);
 
-        createFile(igfsSecondary, FILE, true, BLOCK_SIZE, chunk);
+        createFile(igfsSecondary, FILE, true, /*BLOCK_SIZE,*/ chunk);
 
         appendFile(igfs, FILE, chunk);
 
@@ -1364,10 +1359,10 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest
{
         create(igfsSecondary, paths(DIR, SUBDIR), null);
         create(igfs, null, null);
 
-        igfsSecondary.update(DIR, propsDir);
-        igfsSecondary.update(SUBDIR, propsSubDir);
+        igfsSecondaryFileSystem.update(DIR, propsDir);
+        igfsSecondaryFileSystem.update(SUBDIR, propsSubDir);
 
-        createFile(igfsSecondary, FILE, true, BLOCK_SIZE, chunk);
+        createFile(igfsSecondary, FILE, true, /*BLOCK_SIZE,*/ chunk);
 
         appendFile(igfs, FILE, chunk);
 

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsExUniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsExUniversalFileSystemAdapter.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsExUniversalFileSystemAdapter.java
new file mode 100644
index 0000000..90df1be
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsExUniversalFileSystemAdapter.java
@@ -0,0 +1,71 @@
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.igfs.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * TODO
+ */
+public class IgfsExUniversalFileSystemAdapter implements UniversalFileSystemAdapter {
+
+    private final IgfsEx igfsEx;
+
+    public IgfsExUniversalFileSystemAdapter(IgfsEx igfsEx) {
+        this.igfsEx = igfsEx;
+    }
+
+    @Override public String name() {
+        return igfsEx.name();
+    }
+
+    @Override public boolean exists(String path) {
+        return igfsEx.exists(new IgfsPath(path));
+    }
+
+    @Override public void mkdirs(String path) throws IOException {
+        igfsEx.mkdirs(new IgfsPath(path));
+    }
+
+    @Override public void format() throws IOException {
+        igfsEx.format();
+    }
+
+    @Override public Map<String, String> properties(String path) {
+        return igfsEx.info(new IgfsPath(path)).properties();
+    }
+
+    @Override public void delete(String path, boolean recursive) throws IOException {
+        IgfsPath igfsPath = new IgfsPath(path);
+        boolean del = igfsEx.delete(igfsPath, recursive);
+        if (igfsEx.exists(igfsPath))
+           throw new IOException("Failed to delete, file still exists: " + path);
+
+    }
+
+    @Override public InputStream openInputStream(String path) throws IOException {
+        IgfsPath igfsPath = new IgfsPath(path);
+        IgfsInputStreamAdapter adapter = igfsEx.open(igfsPath);
+        return adapter;
+    }
+
+    @Override public OutputStream openOutputStream(String path, boolean append) throws IOException
{
+        IgfsPath igfsPath = new IgfsPath(path);
+
+        final IgfsOutputStream igfsOutputStream;
+        if (append)
+            igfsOutputStream = igfsEx.append(igfsPath, true/*create*/);
+         else
+            igfsOutputStream = igfsEx.create(igfsPath, true/*overwrite*/);
+
+        return igfsOutputStream;
+    }
+
+    @Override public <T> T getAdapter(Class<T> clazz) {
+        if (clazz == IgfsEx.class)
+            return (T)igfsEx;
+
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
new file mode 100644
index 0000000..e05af74
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/UniversalFileSystemAdapter.java
@@ -0,0 +1,64 @@
+package org.apache.ignite.internal.processors.igfs;
+
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Universal interface to an underlying filesystem.
+ * Typically used for secondary filesystem.
+ * To be used solely in tests.
+ * TODO javadocs
+ */
+public interface UniversalFileSystemAdapter {
+
+    /**
+     * Gets name of the FS.
+     * @return name of this file system.
+     */
+    String name();
+
+    boolean exists(String path) throws IOException;
+
+    void delete(String path, boolean recursive) throws IOException;
+
+    void mkdirs(String path) throws IOException;
+
+    /**
+     * Clears (formats) entire the filesystem.
+     * All the data in the filesystem are DESTROYED.
+     * @throws IOException
+     */
+    void format() throws IOException;
+
+    /**
+     *
+     * @param path
+     * @return
+     */
+    Map<String,String> properties(String path) throws IOException;
+
+    /**
+     * @param path
+     */
+    InputStream openInputStream(String path) throws IOException;
+
+    /**
+     *
+     * @param path
+     * @param append
+     * @return
+     * @throws IOException
+     */
+    OutputStream openOutputStream(String path, boolean append) throws IOException;
+
+    /**
+     *
+     * @param clazz
+     * @param <T>
+     * @return
+     */
+    <T> T getAdapter(Class<T> clazz);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index 11003a5..6d0c386 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -419,4 +419,12 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
             throw new IgniteCheckedException(e);
         }
     }
+
+    /**
+     * Gets the underlying {@link FileSystem}.
+     * @return the underlying Hadoop {@link FileSystem}.
+     */
+    public FileSystem fileSystem() {
+        return fileSys;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop10DualAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop10DualAbstractTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop10DualAbstractTest.java
new file mode 100644
index 0000000..265e6b2
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop10DualAbstractTest.java
@@ -0,0 +1,59 @@
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.*;
+import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.processors.igfs.*;
+
+import java.io.*;
+
+import static org.apache.ignite.igfs.HadoopSecondaryFileSystemConfigurationTest.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Created by ivan on 11.03.15.
+ */
+public abstract class Hadoop10DualAbstractTest extends IgfsDualAbstractSelfTest {
+
+    protected String secondaryConfFullPath;
+    protected String secondaryUri;
+
+    public Hadoop10DualAbstractTest(IgfsMode mode) {
+        super(mode);
+    }
+
+    @Override protected IgfsSecondaryFileSystem createSecondaryFileSystemStack() throws Exception
{
+        startUnderlying();
+
+        prepareConfiguration();
+
+        IgniteHadoopIgfsSecondaryFileSystem second =
+            new IgniteHadoopIgfsSecondaryFileSystem(secondaryUri, secondaryConfFullPath);
+
+        FileSystem fileSystem = second.fileSystem();
+
+        igfsSecondary = new HadoopFileSystemUniversalFileSystemAdapter(fileSystem);
+
+        return second;
+    }
+
+    protected void startUnderlying() throws Exception {
+        Ignite igniteSecondaryUnused
+            = startGridWithIgfs("grid_secondary", "igfs_secondary", PRIMARY, null, SECONDARY_REST_CFG);
+    }
+
+    protected void prepareConfiguration() throws IOException {
+        // TODO: this is borrowed from ConfigTest.
+        // TODO: use port from SECONDARY_REST_CFG.get("port") and better arange parameter
constants.
+        Configuration secondaryConf = configuration(IGFS_SCHEME, SECONDARY_AUTHORITY, true,
true);
+
+        secondaryConf.setInt("fs.igfs.block.size", 1024);
+
+        secondaryConfFullPath = writeConfiguration(secondaryConf, SECONDARY_CFG_PATH);
+
+        secondaryUri = mkUri(IGFS_SCHEME, SECONDARY_AUTHORITY);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualAsyncTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualAsyncTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualAsyncTest.java
new file mode 100644
index 0000000..6737ab6
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualAsyncTest.java
@@ -0,0 +1,12 @@
+package org.apache.ignite.igfs;
+
+/**
+ * Created by ivan on 11.03.15.
+ */
+public class Hadoop1OverIgfsDualAsyncTest extends Hadoop10DualAbstractTest {
+
+    public Hadoop1OverIgfsDualAsyncTest() {
+        super(IgfsMode.DUAL_ASYNC);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualSyncTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualSyncTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualSyncTest.java
new file mode 100644
index 0000000..21f5b6d
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/Hadoop1OverIgfsDualSyncTest.java
@@ -0,0 +1,22 @@
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.ignite.*;
+import org.apache.ignite.internal.processors.igfs.*;
+
+import java.io.*;
+
+import static org.apache.ignite.igfs.HadoopSecondaryFileSystemConfigurationTest.*;
+
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Created by ivan on 11.03.15.
+ */
+public class Hadoop1OverIgfsDualSyncTest extends Hadoop10DualAbstractTest {
+
+    public Hadoop1OverIgfsDualSyncTest() {
+        super(IgfsMode.DUAL_SYNC);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
new file mode 100644
index 0000000..ae8e396
--- /dev/null
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
@@ -0,0 +1,75 @@
+package org.apache.ignite.igfs;
+
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * TODO: javadocs
+ */
+public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFileSystemAdapter
{
+
+    private final FileSystem fileSys;
+
+    public HadoopFileSystemUniversalFileSystemAdapter(FileSystem fs) {
+        this.fileSys = fs;
+    }
+
+    @Override public String name() {
+        return fileSys.getUri().toString();
+    }
+
+    @Override public boolean exists(String path) throws IOException {
+        return fileSys.exists(new Path(path));
+    }
+
+    @Override public void delete(String path, boolean recursive) throws IOException {
+        boolean ok = fileSys.delete(new Path(path), recursive);
+        // TODO: questionable
+        if (!ok)
+            throw new IOException("Failed to delete: " + path);
+    }
+
+    @Override public void mkdirs(String path) throws IOException {
+        boolean ok = fileSys.mkdirs(new Path(path));
+        // TODO: questionable
+        if (!ok)
+            throw new IOException("Failed to mkdirs: " + path);
+    }
+
+    @Override public void format() throws IOException {
+        // TODO: questionable
+        fileSys.delete(new Path("/"), true);
+    }
+
+    @Override public Map<String, String> properties(String path) throws IOException
{
+        Path p = new Path(path);
+        FileStatus status = fileSys.getFileStatus(p);
+        Map<String,String> m = new HashMap<>(3); // max size == 4
+        m.put(IgfsEx.PROP_USER_NAME, status.getOwner());
+        m.put(IgfsEx.PROP_GROUP_NAME, status.getGroup());
+        m.put(IgfsEx.PROP_PERMISSION, status.getPermission().toString());
+        return m;
+    }
+
+    @Override public InputStream openInputStream(String path) throws IOException {
+        return fileSys.open(new Path(path));
+    }
+
+    @Override public OutputStream openOutputStream(String path, boolean append) throws IOException
{
+        Path p = new Path(path);
+        if (append)
+            return fileSys.append(p);
+        else
+            return fileSys.create(p, true/*overwrite*/);
+    }
+
+    @Override public <T> T getAdapter(Class<T> clazz) {
+        if (clazz == FileSystem.class)
+            return (T)fileSys;
+
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/50a789f5/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
index 11279ca..1997ae2 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopSecondaryFileSystemConfigurationTest.java
@@ -54,7 +54,7 @@ import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEnd
  */
 public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstractTest {
     /** IGFS scheme */
-    private static final String IGFS_SCHEME = "igfs";
+    static final String IGFS_SCHEME = "igfs";
 
     /** Primary file system authority. */
     private static final String PRIMARY_AUTHORITY = "igfs:grid0@";
@@ -63,10 +63,10 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
     private static final String PRIMARY_CFG_PATH = "/work/core-site-primary-test.xml";
 
     /** Secondary file system authority. */
-    private static final String SECONDARY_AUTHORITY = "igfs_secondary:grid_secondary@127.0.0.1:11500";
+    static final String SECONDARY_AUTHORITY = "igfs_secondary:grid_secondary@127.0.0.1:11500";
 
     /** Autogenerated secondary file system configuration path. */
-    private static final String SECONDARY_CFG_PATH = "/work/core-site-test.xml";
+    static final String SECONDARY_CFG_PATH = "/work/core-site-test.xml";
 
     /** Secondary endpoint configuration. */
     protected static final Map<String, String> SECONDARY_ENDPOINT_CFG = new HashMap<String,
String>() {{
@@ -470,7 +470,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
      * @param skipLocShmem Whether to skip local shmem mode.
      * @return Configuration.
      */
-    private static Configuration configuration(String scheme, String authority, boolean skipEmbed,
boolean skipLocShmem) {
+    static Configuration configuration(String scheme, String authority, boolean skipEmbed,
boolean skipLocShmem) {
         final Configuration cfg = new Configuration();
 
         if (scheme != null && authority != null)
@@ -494,7 +494,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
      *
      * @param cfg the configuration to set parameters into.
      */
-    private static void setImplClasses(Configuration cfg) {
+    static void setImplClasses(Configuration cfg) {
         cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
 
         cfg.set("fs.AbstractFileSystem.igfs.impl",
@@ -523,7 +523,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
      * @param pathFromIgniteHome path relatively to Ignite home.
      * @return Full path of the written configuration.
      */
-    private static String writeConfiguration(Configuration cfg, String pathFromIgniteHome)
throws IOException {
+    static String writeConfiguration(Configuration cfg, String pathFromIgniteHome) throws
IOException {
         if (!pathFromIgniteHome.startsWith("/"))
             pathFromIgniteHome = "/" + pathFromIgniteHome;
 
@@ -553,7 +553,7 @@ public class HadoopSecondaryFileSystemConfigurationTest extends IgfsCommonAbstra
      * @param authority the authority
      * @return URI String
      */
-    private static String mkUri(String scheme, String authority) {
+    static String mkUri(String scheme, String authority) {
         return scheme + "://" + authority + "/";
     }
 }


Mime
View raw message