camel-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From davscl...@apache.org
Subject [camel] 10/11: CAMEL-12127: Include file-size in file operations API so we can know how big the files are to download/upload.
Date Mon, 08 Jan 2018 10:20:56 GMT
This is an automated email from the ASF dual-hosted git repository.

davsclaus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel.git

commit e9b65f72dcf82c4302810c1f76869119a9ab6c17
Author: Claus Ibsen <claus.ibsen@gmail.com>
AuthorDate: Mon Jan 8 09:25:24 2018 +0100

    CAMEL-12127: Include file-size in file operations API so we can know how big the files
are to download/upload.
---
 .../org/apache/camel/component/file/FileOperations.java   |  6 +++---
 .../apache/camel/component/file/GenericFileConsumer.java  |  2 +-
 .../camel/component/file/GenericFileOperations.java       |  8 +++++---
 .../apache/camel/component/file/GenericFileProducer.java  |  2 +-
 .../file/strategy/GenericFileDeleteProcessStrategy.java   |  4 ++--
 .../file/strategy/GenericFileProcessStrategySupport.java  |  6 +++---
 .../file/strategy/GenericFileRenameProcessStrategy.java   |  2 +-
 .../strategy/GenericFileDeleteProcessStrategyTest.java    |  8 ++++----
 .../file/remote/DefaultFtpClientActivityListener.java     | 15 +++++++++++++--
 .../component/file/remote/FtpClientActivityListener.java  |  2 ++
 .../apache/camel/component/file/remote/FtpOperations.java |  8 +++++---
 .../camel/component/file/remote/SftpOperations.java       |  6 +++---
 .../org/apache/camel/component/scp/ScpOperations.java     |  8 ++++----
 .../camel-example-ftp/src/main/resources/ftp.properties   |  4 ++--
 14 files changed, 49 insertions(+), 32 deletions(-)

diff --git a/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java
b/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java
index 2ee56fe..415f60b 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/FileOperations.java
@@ -202,17 +202,17 @@ public class FileOperations implements GenericFileOperations<File>
{
         return null;
     }
 
-    public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+    public boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException
{
         // noop as we use type converters to read the body content for java.io.File
         return true;
     }
     
     @Override
-    public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
+    public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
         // noop as we used type converters to read the body content for java.io.File
     }
 
-    public boolean storeFile(String fileName, Exchange exchange) throws GenericFileOperationFailedException
{
+    public boolean storeFile(String fileName, Exchange exchange, long size) throws GenericFileOperationFailedException
{
         ObjectHelper.notNull(endpoint, "endpoint");
 
         File file = new File(fileName);
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
index 6a942c1..1cd922f 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileConsumer.java
@@ -406,7 +406,7 @@ public abstract class GenericFileConsumer<T> extends ScheduledBatchPollingConsum
                 boolean retrieved;
                 Exception cause = null;
                 try {
-                    retrieved = operations.retrieveFile(name, exchange);
+                    retrieved = operations.retrieveFile(name, exchange, target.getFileLength());
                 } catch (Exception e) {
                     retrieved = false;
                     cause = e;
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java
b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java
index 23db2a0..9d1b3c8 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileOperations.java
@@ -73,10 +73,11 @@ public interface GenericFileOperations<T> {
      *
      * @param name     name of the file
      * @param exchange stream to write the content of the file into
+     * @param size     the total file size to retrieve, if possible to determine
      * @return true if file has been retrieved, false if not
      * @throws GenericFileOperationFailedException can be thrown
      */
-    boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException;
+    boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException;
     
     /**
      * Releases the resources consumed by a retrieved file
@@ -84,17 +85,18 @@ public interface GenericFileOperations<T> {
      * @param exchange exchange with the content of the file
      * @throws GenericFileOperationFailedException can be thrown
      */
-    void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException;
+    void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException;
 
     /**
      * Stores the content as a new remote file (upload)
      *
      * @param name     name of new file
      * @param exchange with the content content of the file
+     * @param size     the total file size to store, if possible to determine
      * @return true if the file was stored, false if not
      * @throws GenericFileOperationFailedException can be thrown
      */
-    boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException;
+    boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException;
 
     /**
      * Gets the current remote directory
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java
b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java
index ee40b91..2dec738 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/GenericFileProducer.java
@@ -302,7 +302,7 @@ public class GenericFileProducer<T> extends DefaultProducer {
             log.trace("About to write [{}] to [{}] from exchange [{}]", new Object[]{fileName,
getEndpoint(), exchange});
         }
 
-        boolean success = operations.storeFile(fileName, exchange);
+        boolean success = operations.storeFile(fileName, exchange, -1);
         if (!success) {
             throw new GenericFileOperationFailedException("Error writing file [" + fileName
+ "]");
         }
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java
b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java
index 2c6490a..f4aeaeb 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategy.java
@@ -61,7 +61,7 @@ public class GenericFileDeleteProcessStrategy<T> extends GenericFileProcessStrat
 
         try {
             deleteLocalWorkFile(exchange);
-            operations.releaseRetreivedFileResources(exchange);
+            operations.releaseRetrievedFileResources(exchange);
 
             int retries = 3;
             boolean deleted = false;
@@ -101,7 +101,7 @@ public class GenericFileDeleteProcessStrategy<T> extends GenericFileProcessStrat
     public void rollback(GenericFileOperations<T> operations, GenericFileEndpoint<T>
endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
         try {
             deleteLocalWorkFile(exchange);
-            operations.releaseRetreivedFileResources(exchange);
+            operations.releaseRetrievedFileResources(exchange);
 
             // moved the failed file if specifying the moveFailed option
             if (failureRenamer != null) {
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java
b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java
index cbe125e..f38fe67 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileProcessStrategySupport.java
@@ -58,7 +58,7 @@ public abstract class GenericFileProcessStrategySupport<T> implements
GenericFil
 
     public void abort(GenericFileOperations<T> operations, GenericFileEndpoint<T>
endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
         deleteLocalWorkFile(exchange);
-        operations.releaseRetreivedFileResources(exchange);
+        operations.releaseRetrievedFileResources(exchange);
 
         // must release lock last
         if (exclusiveReadLockStrategy != null) {
@@ -68,7 +68,7 @@ public abstract class GenericFileProcessStrategySupport<T> implements
GenericFil
 
     public void commit(GenericFileOperations<T> operations, GenericFileEndpoint<T>
endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
         deleteLocalWorkFile(exchange);
-        operations.releaseRetreivedFileResources(exchange);
+        operations.releaseRetrievedFileResources(exchange);
 
         // must release lock last
         if (exclusiveReadLockStrategy != null) {
@@ -78,7 +78,7 @@ public abstract class GenericFileProcessStrategySupport<T> implements
GenericFil
 
     public void rollback(GenericFileOperations<T> operations, GenericFileEndpoint<T>
endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
         deleteLocalWorkFile(exchange);
-        operations.releaseRetreivedFileResources(exchange);
+        operations.releaseRetrievedFileResources(exchange);
 
         // must release lock last
         if (exclusiveReadLockStrategy != null) {
diff --git a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java
b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java
index 6359f91..ada2a9c 100644
--- a/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java
+++ b/camel-core/src/main/java/org/apache/camel/component/file/strategy/GenericFileRenameProcessStrategy.java
@@ -63,7 +63,7 @@ public class GenericFileRenameProcessStrategy<T> extends GenericFileProcessStrat
     @Override
     public void rollback(GenericFileOperations<T> operations, GenericFileEndpoint<T>
endpoint, Exchange exchange, GenericFile<T> file) throws Exception {
         try {
-            operations.releaseRetreivedFileResources(exchange);
+            operations.releaseRetrievedFileResources(exchange);
 
             if (failureRenamer != null) {
                 // create a copy and bind the file to the exchange to be used by the renamer
to evaluate the file name
diff --git a/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java
b/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java
index ce72224..7367ebd 100644
--- a/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java
+++ b/camel-core/src/test/java/org/apache/camel/component/file/strategy/GenericFileDeleteProcessStrategyTest.java
@@ -67,15 +67,15 @@ public class GenericFileDeleteProcessStrategyTest extends ContextTestSupport
{
             return false;
         }
 
-        public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+        public boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException
{
             return false;
         }
-        @Override
-        public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
+
+        public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
             // No-op
         }
 
-        public boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+        public boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException
{
             return false;
         }
 
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
index 4b45403..333549f 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/DefaultFtpClientActivityListener.java
@@ -29,6 +29,7 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
     private boolean download = true;
 
     private String fileName;
+    private long fileSize;
     private String lastLogActivity;
     private String lastVerboseLogActivity;
     private long lastLogActivityTimestamp = -1;
@@ -52,6 +53,11 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
     }
 
     @Override
+    public void setRemoteFileSize(long fileSize) {
+        this.fileSize = fileSize;
+    }
+
+    @Override
     public String getLastLogActivity() {
         return lastLogActivity;
     }
@@ -124,6 +130,9 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
     public void onBeginDownloading(String host, String file) {
         download = true;
         String msg = "Downloading from host: " + host + " file: " + file + " starting";
+        if (fileSize > 0) {
+            msg += " (file-size: " + fileSize + " bytes)";
+        }
         doLog(msg);
     }
 
@@ -181,10 +190,12 @@ public class DefaultFtpClientActivityListener implements FtpClientActivityListen
 
     @Override
     public void bytesTransferred(long totalBytesTransferred, int bytesTransferred, long streamSize)
{
+        // if stream size is -1 from FTP client then use pre-calculated file size
+        long size = streamSize > 0 ? streamSize : fileSize;
         if (download) {
-            onDownload(host, fileName, bytesTransferred, totalBytesTransferred, streamSize);
+            onDownload(host, fileName, bytesTransferred, totalBytesTransferred, size);
         } else {
-            onUpload(host, fileName, bytesTransferred, totalBytesTransferred, streamSize);
+            onUpload(host, fileName, bytesTransferred, totalBytesTransferred, size);
         }
     }
 
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java
index 07bb0e8..4fb346f 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpClientActivityListener.java
@@ -38,6 +38,8 @@ public interface FtpClientActivityListener extends CopyStreamListener {
 
     void setRemoteFileName(String fileName);
 
+    void setRemoteFileSize(long size);
+
     void onGeneralError(String host, String errorMessage);
 
     void onConnecting(String host);
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
index ea3dc1d..e456158 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/FtpOperations.java
@@ -351,10 +351,11 @@ public class FtpOperations implements RemoteFileOperations<FTPFile>
{
         }
     }
 
-    public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+    public boolean retrieveFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException
{
         // store the name of the file to download on the listener
         clientActivityListener.setDownload(true);
         clientActivityListener.setRemoteFileName(name);
+        clientActivityListener.setRemoteFileSize(size);
         clientActivityListener.onBeginDownloading(endpoint.getConfiguration().remoteServerInformation(),
name);
 
         boolean answer;
@@ -379,7 +380,7 @@ public class FtpOperations implements RemoteFileOperations<FTPFile>
{
     }
     
     @Override
-    public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
+    public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
         InputStream is = exchange.getIn().getHeader(RemoteFileComponent.REMOTE_FILE_INPUT_STREAM,
InputStream.class);
         
         if (is != null) {
@@ -557,13 +558,14 @@ public class FtpOperations implements RemoteFileOperations<FTPFile>
{
         return result;
     }
 
-    public boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+    public boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException
{
         // must normalize name first
         name = endpoint.getConfiguration().normalizePath(name);
 
         // store the name of the file to upload on the listener
         clientActivityListener.setDownload(false);
         clientActivityListener.setRemoteFileName(name);
+        clientActivityListener.setRemoteFileSize(size);
         clientActivityListener.onBeginUploading(endpoint.getConfiguration().remoteServerInformation(),
name);
 
         log.trace("storeFile({})", name);
diff --git a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
index 8604fdf..a06ad5c 100644
--- a/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
+++ b/components/camel-ftp/src/main/java/org/apache/camel/component/file/remote/SftpOperations.java
@@ -646,7 +646,7 @@ public class SftpOperations implements RemoteFileOperations<SftpRemoteFile>
{
         }
     }
 
-    public synchronized boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+    public synchronized boolean retrieveFile(String name, Exchange exchange, long size) throws
GenericFileOperationFailedException {
         LOG.trace("retrieveFile({})", name);
         if (ObjectHelper.isNotEmpty(endpoint.getLocalWorkDirectory())) {
             // local work directory is configured so we should store file content as files
in this local directory
@@ -657,7 +657,7 @@ public class SftpOperations implements RemoteFileOperations<SftpRemoteFile>
{
         }
     }
 
-    public synchronized void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
+    public synchronized void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
         InputStream is = exchange.getIn().getHeader(RemoteFileComponent.REMOTE_FILE_INPUT_STREAM,
InputStream.class);
 
         if (is != null) {
@@ -818,7 +818,7 @@ public class SftpOperations implements RemoteFileOperations<SftpRemoteFile>
{
         return true;
     }
 
-    public synchronized boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+    public synchronized boolean storeFile(String name, Exchange exchange, long size) throws
GenericFileOperationFailedException {
         // must normalize name first
         name = endpoint.getConfiguration().normalizePath(name);
 
diff --git a/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java
b/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java
index efd1eca..3854f4a 100644
--- a/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java
+++ b/components/camel-jsch/src/main/java/org/apache/camel/component/scp/ScpOperations.java
@@ -83,17 +83,17 @@ public class ScpOperations implements RemoteFileOperations<ScpFile>
{
     }
 
     @Override
-    public boolean retrieveFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+    public boolean retrieveFile(String name, Exchange exchange, long isze) throws GenericFileOperationFailedException
{
         return false;
     }
     
     @Override
-    public void releaseRetreivedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
-        // No-op   
+    public void releaseRetrievedFileResources(Exchange exchange) throws GenericFileOperationFailedException
{
+        // noop
     }
 
     @Override
-    public boolean storeFile(String name, Exchange exchange) throws GenericFileOperationFailedException
{
+    public boolean storeFile(String name, Exchange exchange, long size) throws GenericFileOperationFailedException
{
         ObjectHelper.notNull(session, "session");
         ScpConfiguration cfg = endpoint.getConfiguration();
         
diff --git a/examples/camel-example-ftp/src/main/resources/ftp.properties b/examples/camel-example-ftp/src/main/resources/ftp.properties
index 2f1369a..fc42dc4 100644
--- a/examples/camel-example-ftp/src/main/resources/ftp.properties
+++ b/examples/camel-example-ftp/src/main/resources/ftp.properties
@@ -16,10 +16,10 @@
 ## ---------------------------------------------------------------------------
 
 # NOTE: you may need to turn on passive mode via, passiveMode=true
-ftp.client=ftp://changeme-to-ftp-server.com:21/mypath?autoCreate=false&username=bob&password=123
+##ftp.client=ftp://changeme-to-ftp-server.com:21/mypath?autoCreate=false&username=bob&password=123
 
 # this example is a local FTP server
-### ftp.client=ftp://localhost:21?autoCreate=false&username=bob&password=123&passiveMode=true&transferLoggingLevel=INFO
+ftp.client=ftp://localhost:21?autoCreate=false&username=bob&password=123&passiveMode=true&transferLoggingLevel=INFO
 
 # for the server we want to delay 5 seconds between polling the server
 # and keep the downloaded file as-is

-- 
To stop receiving notification emails like this one, please contact
"commits@camel.apache.org" <commits@camel.apache.org>.

Mime
View raw message