camel-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From davscl...@apache.org
Subject [2/2] camel git commit: CAMEL-8266 Allows ignoring directories for SFTP/FTP having no read permission using ignoreFileNotFoundOrPermissionError
Date Sun, 25 Jan 2015 11:41:06 GMT
CAMEL-8266 Allows ignoring directories for SFTP/FTP having no read permission using ignoreFileNotFoundOrPermissionError


Project: http://git-wip-us.apache.org/repos/asf/camel/repo
Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/25d14cf7
Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/25d14cf7
Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/25d14cf7

Branch: refs/heads/camel-2.14.x
Commit: 25d14cf7800eff379806cf590debe1b07f6033df
Parents: 6383558
Author: Shreyas Purohit <shreyas.purohit@gmail.com>
Authored: Sat Jan 24 20:31:14 2015 -0800
Committer: Claus Ibsen <davsclaus@apache.org>
Committed: Sun Jan 25 12:40:52 2015 +0100

----------------------------------------------------------------------
 .../component/file/remote/FtpConsumer.java      |  20 +++-
 .../file/remote/RemoteFileConsumer.java         |  50 +++++++-
 .../component/file/remote/SftpConsumer.java     |   2 +-
 .../remote/RemoteFileIgnoreDoPollErrorTest.java | 118 +++++++++++++++++++
 4 files changed, 181 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/camel/blob/25d14cf7/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
----------------------------------------------------------------------
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
index fff8e93..ba5d32c 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpConsumer.java
@@ -22,6 +22,7 @@ import java.util.List;
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
 import org.apache.camel.component.file.GenericFile;
+import org.apache.camel.component.file.GenericFileOperationFailedException;
 import org.apache.camel.util.FileUtil;
 import org.apache.camel.util.ObjectHelper;
 import org.apache.camel.util.URISupport;
@@ -59,7 +60,7 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> {
     }
 
     protected boolean pollSubDirectory(String absolutePath, String dirName, List<GenericFile<FTPFile>>
fileList, int depth) {
-        boolean answer = doPollDirectory(absolutePath, dirName, fileList, depth);
+        boolean answer = doSafePollSubDirectory(absolutePath, dirName, fileList, depth);
         // change back to parent directory when finished polling sub directory
         if (isStepwise()) {
             operations.changeToParentDirectory();
@@ -167,10 +168,19 @@ public class FtpConsumer extends RemoteFileConsumer<FTPFile> {
     @Override
     protected boolean ignoreCannotRetrieveFile(String name, Exchange exchange, Exception
cause) {
         if (getEndpoint().getConfiguration().isIgnoreFileNotFoundOrPermissionError()) {
-            // error code 550 is file not found
-            int code = exchange.getIn().getHeader(FtpConstants.FTP_REPLY_CODE, 0, int.class);
-            if (code == 550) {
-                return true;
+            if (exchange != null) {
+                // error code 550 is file not found
+                int code = exchange.getIn().getHeader(FtpConstants.FTP_REPLY_CODE, 0, int.class);
+                if (code == 550) {
+                    return true;
+                }
+            }
+            if (cause != null && cause instanceof GenericFileOperationFailedException)
{
+                GenericFileOperationFailedException generic = ObjectHelper.getException(GenericFileOperationFailedException.class,
cause);
+                //exchange is null and cause has the reason for failure to read directories
+                if (generic.getCode() == 550) {
+                    return true;
+                }
             }
         }
         return super.ignoreCannotRetrieveFile(name, exchange, cause);

http://git-wip-us.apache.org/repos/asf/camel/blob/25d14cf7/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
----------------------------------------------------------------------
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
index de242da..bb418e2 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/RemoteFileConsumer.java
@@ -17,9 +17,11 @@
 package org.apache.camel.component.file.remote;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
+import org.apache.camel.component.file.GenericFile;
 import org.apache.camel.component.file.GenericFileConsumer;
 import org.apache.camel.component.file.GenericFileOperationFailedException;
 
@@ -70,13 +72,13 @@ public abstract class RemoteFileConsumer<T> extends GenericFileConsumer<T>
{
             if (!loggedInWarning) {
                 log.warn(message);
                 loggedInWarning = true;
-            } 
+            }
             return false;
         } else {
             // need to log the failed log again
             loggedInWarning = false;
         }
-       
+
         return true;
     }
 
@@ -98,7 +100,7 @@ public abstract class RemoteFileConsumer<T> extends GenericFileConsumer<T>
{
         exchange.setProperty(Exchange.UNIT_OF_WORK_PROCESS_SYNC, Boolean.TRUE);
         return super.processExchange(exchange);
     }
-    
+
     @Override
     protected boolean isRetrieveFile() {
         return getEndpoint().isDownload();
@@ -165,4 +167,46 @@ public abstract class RemoteFileConsumer<T> extends GenericFileConsumer<T>
{
         return ((RemoteFileEndpoint<?>) endpoint).remoteServerInformation();
     }
 
+    /**
+     * Executes doPollDirectory and on exception checks if it can be ignored by calling ignoreCannotRetrieveFile
.
+     *
+     * @param absolutePath The path of the directory to poll
+     * @param dirName The name of the directory to poll
+     * @param fileList current list of files gathered
+     * @param depth the current depth of the directory
+     * @return whether or not to continue polling, <tt>false</tt> means the maxMessagesPerPoll
limit has been hit
+     * @throws GenericFileOperationFailedException if the exception during doPollDirectory
can not be ignored
+     */
+    protected boolean doSafePollSubDirectory(String absolutePath, String dirName, List<GenericFile<T>>
fileList, int depth) {
+        try {
+            log.trace("Polling sub directory: {} from: {}", absolutePath, endpoint);
+            //Try to poll the directory
+            return doPollDirectory(absolutePath, dirName, fileList, depth);
+        } catch (Exception e) {
+            log.debug("Caught exception " + e.getMessage());
+            if (ignoreCannotRetrieveFile(absolutePath, null, e)) {
+                log.trace("Ignoring file error " + e.getMessage() + " for " + absolutePath);
+                //indicate no files in this directory to poll, continue with fileList
+                return true;
+            } else {
+                log.trace("Not ignoring file error " + e.getMessage() + " for " + absolutePath);
+                if (e instanceof GenericFileOperationFailedException) {
+                    throw (GenericFileOperationFailedException) e;
+                } else {
+                    throw new GenericFileOperationFailedException("Cannot poll sub-directory:
" + absolutePath + " from: " + endpoint, e);
+                }
+            }
+        }
+    }
+
+    /**
+     * Poll directory given by dirName or absolutePath
+     *
+     * @param absolutePath The path of the directory to poll
+     * @param dirName The name of the directory to poll
+     * @param fileList current list of files gathered
+     * @param depth the current depth of the directory
+     * @return whether or not to continue polling, <tt>false</tt> means the maxMessagesPerPoll
limit has been hit
+     */
+    protected abstract boolean doPollDirectory(String absolutePath, String dirName, List<GenericFile<T>>
fileList, int depth);
 }

http://git-wip-us.apache.org/repos/asf/camel/blob/25d14cf7/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
----------------------------------------------------------------------
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
index 0d0906b..0f2f536 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpConsumer.java
@@ -58,7 +58,7 @@ public class SftpConsumer extends RemoteFileConsumer<ChannelSftp.LsEntry>
{
     }
 
     protected boolean pollSubDirectory(String absolutePath, String dirName, List<GenericFile<ChannelSftp.LsEntry>>
fileList, int depth) {
-        boolean answer = doPollDirectory(absolutePath, dirName, fileList, depth);
+        boolean answer = doSafePollSubDirectory(absolutePath, dirName, fileList, depth);
         // change back to parent directory when finished polling sub directory
         if (isStepwise()) {
             operations.changeToParentDirectory();

http://git-wip-us.apache.org/repos/asf/camel/blob/25d14cf7/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java
----------------------------------------------------------------------
diff --git a/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java
b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java
new file mode 100644
index 0000000..3ca70b6
--- /dev/null
+++ b/components/camel-ftp/src/test/java/org/apache/camel/component/file/remote/RemoteFileIgnoreDoPollErrorTest.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.file.remote;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.component.file.GenericFile;
+import org.apache.camel.component.file.GenericFileOperationFailedException;
+import org.apache.camel.component.file.GenericFileProducer;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class RemoteFileIgnoreDoPollErrorTest {
+    private final RemoteFileEndpoint<Object> remoteFileEndpoint = new RemoteFileEndpoint<Object>()
{
+        @Override
+        protected RemoteFileConsumer<Object> buildConsumer(Processor processor) {
+            return null;
+        }
+
+        @Override
+        protected GenericFileProducer<Object> buildProducer() {
+            return null;
+        }
+
+        @Override
+        public RemoteFileOperations<Object> createRemoteFileOperations() throws Exception
{
+            return null;
+        }
+
+        @Override
+        public String getScheme() {
+            return null;
+        }
+    };
+
+    @Test
+    public void testReadDirErrorIsHandled() throws Exception {
+        RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("true", true);
+        boolean result = consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(),
0);
+        Assert.assertTrue(result);
+    }
+
+    @Test
+    public void testReadDirErrorIsHandledWithNoMorePoll() throws Exception {
+        RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("false", true);
+        boolean result = consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(),
0);
+        Assert.assertFalse(result);
+    }
+
+    @Test
+    public void testReadDirErrorNotHandled() throws Exception {
+        RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("IllegalStateException",
false);
+        try {
+            consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(),
0);
+            Assert.fail("Must throw wrapped IllegalStateException in GenericFileOperationFailedException");
+        } catch (GenericFileOperationFailedException e) {
+            Assert.assertTrue(e.getCause() instanceof IllegalStateException);
+        }
+    }
+
+    @Test
+    public void testReadDirErrorNotHandledForGenericFileOperationException() throws Exception
{
+        RemoteFileConsumer<Object> consumer = getRemoteFileConsumer("GenericFileOperationFailedException",
false);
+        try {
+            consumer.doSafePollSubDirectory("anyPath", "adir", new ArrayList<GenericFile<Object>>(),
0);
+            Assert.fail("Must throw GenericFileOperationFailedException");
+        } catch (GenericFileOperationFailedException e) {
+            Assert.assertNull(e.getCause());
+        }
+    }
+
+    private RemoteFileConsumer<Object> getRemoteFileConsumer(final String doPollResult,
final boolean ignoreCannotRetrieveFile) {
+        return new RemoteFileConsumer<Object>(remoteFileEndpoint, null, null) {
+            @Override
+            protected boolean doPollDirectory(String absolutePath, String dirName, List<GenericFile<Object>>
genericFiles, int depth) {
+                switch (doPollResult) {
+                case "IllegalStateException":
+                    throw new IllegalStateException("Problem");
+                case "GenericFileOperationFailedException":
+                    throw new GenericFileOperationFailedException("Perm error");
+                case "true":
+                    return true;
+                default:
+                    return false;
+                }
+            }
+            @Override
+            protected boolean pollDirectory(String fileName, List<GenericFile<Object>>
genericFiles, int depth) {
+                return false;
+            }
+            @Override
+            protected boolean isMatched(GenericFile<Object> file, String doneFileName,
List<Object> files) {
+                return false;
+            }
+            @Override
+            protected boolean ignoreCannotRetrieveFile(String name, Exchange exchange, Exception
cause) {
+                return ignoreCannotRetrieveFile;
+            }
+        };
+    }
+}


Mime
View raw message