streams-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mfrank...@apache.org
Subject [1/2] git commit: STREAMS-166 | Modified the S3PeristReader to check for object summaries in the listing response. If there are any object summaries then we can assume that the path is pointing to a directory and we need to iterate through all files that
Date Wed, 24 Sep 2014 17:46:40 GMT
Repository: incubator-streams
Updated Branches:
  refs/heads/master 8c45159b0 -> a9f1b2fed


STREAMS-166 | Modified the S3PeristReader to check for object summaries in the listing response.
If there are any object summaries then we can assume that the path is pointing to a directory
and we need to iterate through all files that it contains.


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/8fb07eb2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/8fb07eb2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/8fb07eb2

Branch: refs/heads/master
Commit: 8fb07eb2f3ebfae42f612ad169c06115914d8a4f
Parents: 3ee5175
Author: Robert Douglas <rdouglas@w2odigital.com>
Authored: Tue Sep 9 15:32:50 2014 -0500
Committer: Robert Douglas <rdouglas@w2odigital.com>
Committed: Tue Sep 9 15:32:50 2014 -0500

----------------------------------------------------------------------
 .../org/apache/streams/s3/S3PersistReader.java   | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8fb07eb2/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
index 1123dbb..4f62a06 100644
--- a/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
+++ b/streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3PersistReader.java
@@ -25,6 +25,7 @@ import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.S3ClientOptions;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Queues;
 import org.apache.streams.core.*;
@@ -116,7 +117,7 @@ public class S3PersistReader implements StreamsPersistReader, DatumStatusCountab
         final ListObjectsRequest request = new ListObjectsRequest()
                 .withBucketName(this.s3ReaderConfiguration.getBucket())
                 .withPrefix(s3ReaderConfiguration.getReaderPath())
-                .withMaxKeys(50);
+                .withMaxKeys(500);
 
 
         ObjectListing listing = this.amazonS3Client.listObjects(request);
@@ -128,12 +129,22 @@ public class S3PersistReader implements StreamsPersistReader, DatumStatusCountab
          * if you cannot list files that are in this path, then you are most likely dealing
with
          * a simple file.
          */
-        if(listing.getCommonPrefixes().size() > 0) {
+        boolean hasCommonPrefixes = listing.getCommonPrefixes().size() > 0 ? true : false;
+        boolean hasObjectSummaries = listing.getObjectSummaries().size() > 0 ? true :
false;
+
+        if(hasCommonPrefixes || hasObjectSummaries) {
             // Handle the 'directory' use case
             do
             {
-                for (String file : listing.getCommonPrefixes())
-                    this.files.add(file);
+                if(hasCommonPrefixes) {
+                    for (String file : listing.getCommonPrefixes()) {
+                        this.files.add(file);
+                    }
+                } else {
+                    for(final S3ObjectSummary objectSummary : listing.getObjectSummaries())
{
+                        this.files.add(objectSummary.getKey());
+                    }
+                }
 
                 // get the next batch.
                 listing = this.amazonS3Client.listNextBatchOfObjects(listing);


Mime
View raw message