apex-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yogidevendra <...@git.apache.org>
Subject [GitHub] apex-malhar pull request #360: APEXMALHAR-2174-S3-ReaderIssue Fixed the S3 r...
Date Thu, 04 Aug 2016 06:11:46 GMT
Github user yogidevendra commented on a diff in the pull request:

    https://github.com/apache/apex-malhar/pull/360#discussion_r73466833
  
    --- Diff: library/src/main/java/com/datatorrent/lib/io/fs/S3BlockReader.java ---
    @@ -47,63 +56,144 @@ public S3BlockReader()
       public void setup(Context.OperatorContext context)
       {
         super.setup(context);
    -    s3bucketUri = fs.getScheme() + "://" + bucketName;
    +    s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
    +    ((S3BlockReaderContext)readerContext).setBucketName(bucketName);
    +    ((S3BlockReaderContext)readerContext).setS3Client(s3Client);
       }
     
       /**
        * Extracts the bucket name from the given uri
        * @param s3uri s3 uri
        * @return name of the bucket
        */
    -  @VisibleForTesting
       protected static String extractBucket(String s3uri)
       {
         return s3uri.substring(s3uri.indexOf('@') + 1, s3uri.indexOf("/", s3uri.indexOf('@')));
       }
     
       /**
    -   * Create the stream from the bucket uri and block path.
    +   * Extracts the accessKey from the given uri
    +   * @param s3uri given s3 uri
    +   * @return the accessKey
    +   */
    +  protected static String extractAccessKey(String s3uri)
    +  {
    +    return s3uri.substring(s3uri.indexOf("://") + 3, s3uri.indexOf(':', s3uri.indexOf("://")
+ 3));
    +  }
    +
    +  /**
    +   * Extracts the secretAccessKey from the given uri
    +   * @param s3uri given s3uri
    +   * @return the secretAccessKey
    +   */
    +  protected static String extractSecretAccessKey(String s3uri)
    +  {
    +    return s3uri.substring(s3uri.indexOf(':', s3uri.indexOf("://") + 1) + 1, s3uri.indexOf('@'));
    +  }
    +
    +  /**
    +   * Extract the file path from given block and set it to the readerContext
        * @param block block metadata
        * @return stream
        * @throws IOException
        */
       @Override
       protected FSDataInputStream setupStream(BlockMetadata.FileBlockMetadata block) throws
IOException
       {
    -    FSDataInputStream ins = fs.open(new Path(s3bucketUri + block.getFilePath()));
    -    ins.seek(block.getOffset());
    -    return ins;
    +    String filePath = block.getFilePath();
    +    if (filePath.startsWith("/")) {
    --- End diff --
    
    Please add comments mentioning why this is needed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message