ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From voze...@apache.org
Subject [12/12] ignite git commit: IGNITE-2206: Fixed a bug with file system factory deserialization.
Date Wed, 23 Dec 2015 14:17:10 GMT
IGNITE-2206: Fixed a bug with file system factory deserialization.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/04827ca7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/04827ca7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/04827ca7

Branch: refs/heads/ignite-2206
Commit: 04827ca7f2f96d50e890b98603f1a9f41e1c853a
Parents: 061c222
Author: vozerov-gridgain <vozerov@gridgain.com>
Authored: Wed Dec 23 17:17:40 2015 +0300
Committer: vozerov-gridgain <vozerov@gridgain.com>
Committed: Wed Dec 23 17:17:40 2015 +0300

----------------------------------------------------------------------
 .../internal/processors/igfs/IgfsPaths.java     | 55 +++++++++++++-------
 .../hadoop/fs/v1/IgniteHadoopFileSystem.java    | 35 ++++++++-----
 .../hadoop/fs/v2/IgniteHadoopFileSystem.java    | 41 +++++++++------
 .../hadoop/fs/HadoopLazyConcurrentMap.java      |  4 --
 4 files changed, 81 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index cd34655..0bb581b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -27,6 +27,8 @@ import java.io.ObjectOutput;
 import java.io.ObjectOutputStream;
 import java.util.ArrayList;
 import java.util.List;
+
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.igfs.IgfsMode;
 import org.apache.ignite.igfs.IgfsPath;
 import org.apache.ignite.internal.util.typedef.T2;
@@ -41,7 +43,7 @@ public class IgfsPaths implements Externalizable {
     private static final long serialVersionUID = 0L;
 
     /** */
-    private Object payload;
+    private byte[] payloadBytes;
 
     /** Default IGFS mode. */
     private IgfsMode dfltMode;
@@ -62,11 +64,27 @@ public class IgfsPaths implements Externalizable {
      * @param payload Payload.
      * @param dfltMode Default IGFS mode.
      * @param pathModes Path modes.
+     * @throws IgniteCheckedException If failed.
      */
-    public IgfsPaths(Object payload, IgfsMode dfltMode, @Nullable List<T2<IgfsPath,
IgfsMode>> pathModes) {
-        this.payload = payload;
+    public IgfsPaths(Object payload, IgfsMode dfltMode, @Nullable List<T2<IgfsPath,
IgfsMode>> pathModes)
+        throws IgniteCheckedException {
         this.dfltMode = dfltMode;
         this.pathModes = pathModes;
+
+        if (payload == null)
+            payloadBytes = null;
+        else {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+            try (ObjectOutput oo = new ObjectOutputStream(baos)) {
+                oo.writeObject(payload);
+            }
+            catch (IOException e) {
+                throw new IgniteCheckedException("Failed to serialize secondary file system
factory: " + payload, e);
+            }
+
+            payloadBytes = baos.toByteArray();
+        }
     }
 
     /**
@@ -85,20 +103,25 @@ public class IgfsPaths implements Externalizable {
 
     /**
      * @return Payload.
+     *
+     * @throws IgniteCheckedException If failed to deserialize the payload.
      */
-    @Nullable public Object getPayload() {
-        return payload;
+    @Nullable public Object getPayload() throws IgniteCheckedException {
+        if (payloadBytes == null)
+            return null;
+        else {
+            try (ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(payloadBytes)))
{
+                return oi.readObject();
+            }
+            catch (IOException | ClassNotFoundException e) {
+                throw new IgniteCheckedException("Failed to deserialize secondary file system
factory. ", e);
+            }
+        }
     }
 
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
-        try (ObjectOutput oo = new ObjectOutputStream(baos)) {
-            oo.writeObject(payload);
-        }
-
-        U.writeByteArray(out, baos.toByteArray());
+        U.writeByteArray(out, payloadBytes);
 
         U.writeEnum(out, dfltMode);
 
@@ -120,13 +143,7 @@ public class IgfsPaths implements Externalizable {
 
     /** {@inheritDoc} */
     @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException
{
-        byte[] factoryBytes = U.readByteArray(in);
-
-        assert factoryBytes != null;
-
-        try (ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes)))
{
-            payload = oi.readObject();
-        }
+        payloadBytes = U.readByteArray(in);
 
         dfltMode = IgfsMode.fromOrdinal(in.readByte());
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 545c905..20cd9ca 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -17,19 +17,6 @@
 
 package org.apache.ignite.hadoop.fs.v1;
 
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.ContentSummary;
@@ -43,6 +30,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
 import org.apache.ignite.igfs.IgfsBlockLocation;
@@ -71,6 +59,20 @@ import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lifecycle.LifecycleAware;
 import org.jetbrains.annotations.Nullable;
 
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_BATCH_SIZE;
 import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_DIR;
 import static org.apache.ignite.igfs.IgfsMode.PROXY;
@@ -329,7 +331,12 @@ public class IgniteHadoopFileSystem extends FileSystem {
             }
 
             if (initSecondary) {
-                factory = (HadoopFileSystemFactory)paths.getPayload();
+                try {
+                    factory = (HadoopFileSystemFactory) paths.getPayload();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IOException("Failed to get secondary file system factory.",
e);
+                }
 
                 A.ensure(factory != null, "Secondary file system factory should not be null.");
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index d665d4c..59d870c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -17,22 +17,6 @@
 
 package org.apache.ignite.hadoop.fs.v2;
 
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -51,6 +35,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.Progressable;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.igfs.IgfsBlockLocation;
 import org.apache.ignite.igfs.IgfsFile;
@@ -76,6 +61,23 @@ import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lifecycle.LifecycleAware;
 import org.jetbrains.annotations.Nullable;
 
+import java.io.BufferedOutputStream;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_BATCH_SIZE;
 import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_DIR;
 import static org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.getFsHadoopUser;
@@ -336,7 +338,12 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements
Closea
             }
 
             if (initSecondary) {
-                factory = (HadoopAbstractFileSystemFactory)paths.getPayload();
+                try {
+                    factory = (HadoopAbstractFileSystemFactory) paths.getPayload();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IOException("Failed to get secondary file system factory.",
e);
+                }
 
                 A.ensure(secondaryUri != null, "File system factory uri should not be null.");
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
index 2b20639..5808d01 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
@@ -112,10 +112,6 @@ public class HadoopLazyConcurrentMap<K, V extends Closeable> {
             if (closed)
                 return;
 
-            // TODO: debug:
-            System.out.println("##### closed: " + System.identityHashCode(this));
-            Thread.dumpStack();
-
             closed = true;
 
             Exception err = null;


Mime
View raw message