streams-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mfranklin <...@git.apache.org>
Subject [GitHub] incubator-streams pull request: Twitter Modificaitons
Date Mon, 05 May 2014 21:36:52 GMT
Github user mfranklin commented on a diff in the pull request:

    https://github.com/apache/incubator-streams/pull/8#discussion_r12301195
  
    --- Diff: streams-contrib/streams-amazon-aws/streams-persist-s3/src/main/java/org/apache/streams/s3/S3OutputStreamWrapper.java
---
    @@ -0,0 +1,128 @@
    +package org.apache.streams.s3;
    +
    +import com.amazonaws.services.s3.AmazonS3Client;
    +import com.amazonaws.services.s3.model.ObjectMetadata;
    +import com.amazonaws.services.s3.transfer.TransferManager;
    +import com.amazonaws.services.s3.transfer.Upload;
    +import org.joda.time.DateTime;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import java.io.*;
    +
    +import com.amazonaws.services.s3.AmazonS3Client;
    +import com.amazonaws.services.s3.model.ObjectMetadata;
    +import com.amazonaws.services.s3.model.PutObjectResult;
    +import com.amazonaws.services.s3.transfer.TransferManager;
    +import com.amazonaws.services.s3.transfer.Upload;
    +import org.apache.commons.io.FilenameUtils;
    +import org.joda.time.DateTime;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import java.io.*;
    +import java.util.Date;
    +import java.util.Map;
    +
    +/**
    + *
    + * Author:  Smashew
    + * Date:    2014-04-14
    + *
    + * Description:
    + * This class uses ByteArrayOutputStreams to ensure files are written to S3 properly.
    + *
    + * There is a way to upload data in chunks (5mb or so) a peice, but the multi-part upload
    + * is kind of a PITA to deal with.
    + *
    + * // TODO: This should be refactored to allow a user to specify if they want one large
file instead of many small ones
    + */
    +public class S3OutputStreamWrapper extends OutputStream
    +{
    +    private static final Logger LOGGER = LoggerFactory.getLogger(S3OutputStreamWrapper.class);
    +
    +    private final AmazonS3Client amazonS3Client;
    +    private final String bucketName;
    +    private final String path;
    +    private final String fileName;
    +    private ByteArrayOutputStream outputStream;
    +    private final Map<String, String> metaData;
    +
    +    private boolean isClosed = false;
    +
    +    public S3OutputStreamWrapper(AmazonS3Client amazonS3Client, String bucketName, String
path, String fileName, Map<String, String> metaData) throws IOException
    +    {
    +        this.amazonS3Client = amazonS3Client;
    +        this.bucketName = bucketName;
    +        this.path = path;
    +        this.fileName = fileName;
    +        this.metaData = metaData;
    +        this.outputStream = new ByteArrayOutputStream();
    +    }
    +
    +    /*
    +     * The Methods that are overriden to support the 'OutputStream' object.
    +     */
    +
    +    public void write(int b) throws IOException                         { this.outputStream.write(b);
}
    --- End diff --
    
    Formatting


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message