Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 194BA10D80 for ; Wed, 6 Nov 2013 08:07:42 +0000 (UTC) Received: (qmail 29523 invoked by uid 500); 6 Nov 2013 08:07:40 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 29476 invoked by uid 500); 6 Nov 2013 08:07:40 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 29469 invoked by uid 99); 6 Nov 2013 08:07:40 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Nov 2013 08:07:40 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Nov 2013 08:07:35 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 8DF5723889FA; Wed, 6 Nov 2013 08:07:12 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1539259 - in /hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/s3/ src/main/java/org/apache/hadoop/fs/s3native/ Date: Wed, 06 Nov 2013 08:07:12 -0000 To: common-commits@hadoop.apache.org From: cmccabe@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131106080712.8DF5723889FA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cmccabe Date: Wed Nov 6 08:07:11 2013 New Revision: 1539259 URL: http://svn.apache.org/r1539259 Log: HADOOP-9623. Update jet3t dependency to 0.9.0 (Amandeep Khurana via Colin Patrick McCabe) Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1539259&r1=1539258&r2=1539259&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Nov 6 08:07:11 2013 @@ -14,6 +14,9 @@ Release 2.2.1 - UNRELEASED HADOOP-10079. log a warning message if group resolution takes too long. (cmccabe) + HADOOP-9623. Update jets3t dependency to 0.9.0 (Amandeep Khurana via Colin + Patrick McCabe) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java?rev=1539259&r1=1539258&r2=1539259&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java Wed Nov 6 08:07:11 2013 @@ -42,6 +42,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3.INode.FileType; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; +import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; @@ -60,8 +61,8 @@ class Jets3tFileSystemStore implements F private static final String FILE_SYSTEM_VERSION_NAME = "fs-version"; private static final String FILE_SYSTEM_VERSION_VALUE = "1"; - private static final Map METADATA = - new HashMap(); + private static final Map METADATA = + new HashMap(); static { METADATA.put(FILE_SYSTEM_NAME, FILE_SYSTEM_VALUE); @@ -159,7 +160,7 @@ class Jets3tFileSystemStore implements F throws IOException { try { - S3Object object = s3Service.getObject(bucket, key); + S3Object object = s3Service.getObject(bucket.getName(), key); if (checkMetadata) { checkMetadata(object); } @@ -172,6 +173,9 @@ class Jets3tFileSystemStore implements F throw (IOException) e.getCause(); } throw new S3Exception(e); + } catch (ServiceException e) { + handleServiceException(e); + return null; } } @@ -188,6 +192,9 @@ class Jets3tFileSystemStore implements F throw (IOException) e.getCause(); } throw new S3Exception(e); + } catch (ServiceException e) { + handleServiceException(e); + return null; } } @@ -264,7 +271,7 @@ class Jets3tFileSystemStore implements F if (!prefix.endsWith(PATH_DELIMITER)) { prefix += PATH_DELIMITER; } - S3Object[] objects = s3Service.listObjects(bucket, prefix, PATH_DELIMITER); + S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, PATH_DELIMITER); Set prefixes = new TreeSet(); for (int i = 0; i < objects.length; i++) { prefixes.add(keyToPath(objects[i].getKey())); @@ -286,7 +293,7 @@ class Jets3tFileSystemStore implements F if (!prefix.endsWith(PATH_DELIMITER)) { prefix += PATH_DELIMITER; } - S3Object[] objects = s3Service.listObjects(bucket, prefix, null); + S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null); Set prefixes = new TreeSet(); for (int i = 0; i < objects.length; i++) { prefixes.add(keyToPath(objects[i].getKey())); @@ -369,7 +376,7 @@ class Jets3tFileSystemStore implements F @Override public void purge() throws IOException { try { - S3Object[] objects = s3Service.listObjects(bucket); + S3Object[] objects = s3Service.listObjects(bucket.getName()); for (int i = 0; i < objects.length; i++) { s3Service.deleteObject(bucket, objects[i].getKey()); } @@ -386,7 +393,7 @@ class Jets3tFileSystemStore implements F StringBuilder sb = new StringBuilder("S3 Filesystem, "); sb.append(bucket.getName()).append("\n"); try { - S3Object[] objects = s3Service.listObjects(bucket, PATH_DELIMITER, null); + S3Object[] objects = s3Service.listObjects(bucket.getName(), PATH_DELIMITER, null); for (int i = 0; i < objects.length; i++) { Path path = keyToPath(objects[i].getKey()); sb.append(path).append("\n"); @@ -408,4 +415,15 @@ class Jets3tFileSystemStore implements F System.out.println(sb); } + private void handleServiceException(ServiceException e) throws IOException { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + else { + if(LOG.isDebugEnabled()) { + LOG.debug("Got ServiceException with Error code: " + e.getErrorCode() + ";and Error message: " + e.getErrorMessage()); + } + } + } + } Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java?rev=1539259&r1=1539258&r2=1539259&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/MigrationTool.java Wed Nov 6 08:07:11 2013 @@ -34,6 +34,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; +import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; @@ -177,7 +178,7 @@ public class MigrationTool extends Confi private S3Object get(String key) { try { - return s3Service.getObject(bucket, key); + return s3Service.getObject(bucket.getName(), key); } catch (S3ServiceException e) { if ("NoSuchKey".equals(e.getS3ErrorCode())) { return null; @@ -200,7 +201,7 @@ public class MigrationTool extends Confi public Set listAllPaths() throws IOException { try { String prefix = urlEncode(Path.SEPARATOR); - S3Object[] objects = s3Service.listObjects(bucket, prefix, null); + S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null); Set prefixes = new TreeSet(); for (int i = 0; i < objects.length; i++) { prefixes.add(keyToPath(objects[i].getKey())); @@ -237,7 +238,7 @@ public class MigrationTool extends Confi private InputStream get(String key) throws IOException { try { - S3Object object = s3Service.getObject(bucket, key); + S3Object object = s3Service.getObject(bucket.getName(), key); return object.getDataInputStream(); } catch (S3ServiceException e) { if ("NoSuchKey".equals(e.getS3ErrorCode())) { @@ -247,6 +248,8 @@ public class MigrationTool extends Confi throw (IOException) e.getCause(); } throw new S3Exception(e); + } catch (ServiceException e) { + return null; } } Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java?rev=1539259&r1=1539258&r2=1539259&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java Wed Nov 6 08:07:11 2013 @@ -29,17 +29,21 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3.S3Credentials; import org.apache.hadoop.fs.s3.S3Exception; -import org.jets3t.service.S3ObjectsChunk; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; +import org.jets3t.service.ServiceException; +import org.jets3t.service.StorageObjectsChunk; import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.model.S3Bucket; import org.jets3t.service.model.S3Object; +import org.jets3t.service.model.StorageObject; import org.jets3t.service.security.AWSCredentials; @InterfaceAudience.Private @@ -48,7 +52,9 @@ class Jets3tNativeFileSystemStore implem private S3Service s3Service; private S3Bucket bucket; - + public static final Log LOG = + LogFactory.getLog(Jets3tNativeFileSystemStore.class); + @Override public void initialize(URI uri, Configuration conf) throws IOException { S3Credentials s3Credentials = new S3Credentials(); @@ -59,7 +65,7 @@ class Jets3tNativeFileSystemStore implem s3Credentials.getSecretAccessKey()); this.s3Service = new RestS3Service(awsCredentials); } catch (S3ServiceException e) { - handleServiceException(e); + handleS3ServiceException(e); } bucket = new S3Bucket(uri.getHost()); } @@ -80,7 +86,7 @@ class Jets3tNativeFileSystemStore implem } s3Service.putObject(bucket, object); } catch (S3ServiceException e) { - handleServiceException(e); + handleS3ServiceException(e); } finally { if (in != null) { try { @@ -101,53 +107,85 @@ class Jets3tNativeFileSystemStore implem object.setContentLength(0); s3Service.putObject(bucket, object); } catch (S3ServiceException e) { - handleServiceException(e); + handleS3ServiceException(e); } } @Override public FileMetadata retrieveMetadata(String key) throws IOException { try { - S3Object object = s3Service.getObjectDetails(bucket, key); + if(LOG.isDebugEnabled()) { + LOG.debug("Getting metadata for key: " + key + " from bucket:" + bucket.getName()); + } + S3Object object = s3Service.getObject(bucket.getName(), key); return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime()); } catch (S3ServiceException e) { // Following is brittle. Is there a better way? - if (e.getMessage().contains("ResponseCode=404")) { - return null; + if (e.getS3ErrorCode().matches("NoSuchKey")) { + return null; //return null if key not found } - handleServiceException(e); + handleS3ServiceException(e); return null; //never returned - keep compiler happy } } - + + /** + * @param key + * The key is the object name that is being retrieved from the S3 bucket + * @return + * This method returns null if the key is not found + * @throws IOException + */ + @Override public InputStream retrieve(String key) throws IOException { try { - S3Object object = s3Service.getObject(bucket, key); + if(LOG.isDebugEnabled()) { + LOG.debug("Getting key: " + key + " from bucket:" + bucket.getName()); + } + S3Object object = s3Service.getObject(bucket.getName(), key); return object.getDataInputStream(); } catch (S3ServiceException e) { - handleServiceException(key, e); + handleS3ServiceException(key, e); return null; //never returned - keep compiler happy + } catch (ServiceException e) { + handleServiceException(e); + return null; //return null if key not found } } - + + /** + * + * @param key + * The key is the object name that is being retrieved from the S3 bucket + * @return + * This method returns null if the key is not found + * @throws IOException + */ + @Override public InputStream retrieve(String key, long byteRangeStart) - throws IOException { + throws IOException { try { + if(LOG.isDebugEnabled()) { + LOG.debug("Getting key: " + key + " from bucket:" + bucket.getName() + " with byteRangeStart: " + byteRangeStart); + } S3Object object = s3Service.getObject(bucket, key, null, null, null, null, byteRangeStart, null); return object.getDataInputStream(); } catch (S3ServiceException e) { - handleServiceException(key, e); + handleS3ServiceException(key, e); return null; //never returned - keep compiler happy + } catch (ServiceException e) { + handleServiceException(e); + return null; //return null if key not found } } @Override public PartialListing list(String prefix, int maxListingLength) - throws IOException { + throws IOException { return list(prefix, maxListingLength, null, false); } @@ -158,6 +196,13 @@ class Jets3tNativeFileSystemStore implem return list(prefix, recurse ? null : PATH_DELIMITER, maxListingLength, priorLastKey); } + /** + * + * @return + * This method returns null if the list could not be populated + * due to S3 giving ServiceException + * @throws IOException + */ private PartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey) throws IOException { @@ -165,52 +210,63 @@ class Jets3tNativeFileSystemStore implem if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) { prefix += PATH_DELIMITER; } - S3ObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), + StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), prefix, delimiter, maxListingLength, priorLastKey); FileMetadata[] fileMetadata = new FileMetadata[chunk.getObjects().length]; for (int i = 0; i < fileMetadata.length; i++) { - S3Object object = chunk.getObjects()[i]; + StorageObject object = chunk.getObjects()[i]; fileMetadata[i] = new FileMetadata(object.getKey(), object.getContentLength(), object.getLastModifiedDate().getTime()); } return new PartialListing(chunk.getPriorLastKey(), fileMetadata, chunk.getCommonPrefixes()); } catch (S3ServiceException e) { - handleServiceException(e); + handleS3ServiceException(e); return null; //never returned - keep compiler happy + } catch (ServiceException e) { + handleServiceException(e); + return null; //return null if list could not be populated } } @Override public void delete(String key) throws IOException { try { + if(LOG.isDebugEnabled()) { + LOG.debug("Deleting key:" + key + "from bucket" + bucket.getName()); + } s3Service.deleteObject(bucket, key); } catch (S3ServiceException e) { - handleServiceException(key, e); + handleS3ServiceException(key, e); } } @Override public void copy(String srcKey, String dstKey) throws IOException { try { + if(LOG.isDebugEnabled()) { + LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName()); + } s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(), new S3Object(dstKey), false); } catch (S3ServiceException e) { - handleServiceException(srcKey, e); + handleS3ServiceException(srcKey, e); + } catch (ServiceException e) { + handleServiceException(e); } } @Override public void purge(String prefix) throws IOException { try { - S3Object[] objects = s3Service.listObjects(bucket, prefix, null); + S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null); for (S3Object object : objects) { s3Service.deleteObject(bucket, object.getKey()); } } catch (S3ServiceException e) { - handleServiceException(e); + handleS3ServiceException(e); } } @@ -219,30 +275,44 @@ class Jets3tNativeFileSystemStore implem StringBuilder sb = new StringBuilder("S3 Native Filesystem, "); sb.append(bucket.getName()).append("\n"); try { - S3Object[] objects = s3Service.listObjects(bucket); + S3Object[] objects = s3Service.listObjects(bucket.getName()); for (S3Object object : objects) { sb.append(object.getKey()).append("\n"); } } catch (S3ServiceException e) { - handleServiceException(e); + handleS3ServiceException(e); } System.out.println(sb); } - private void handleServiceException(String key, S3ServiceException e) throws IOException { + private void handleS3ServiceException(String key, S3ServiceException e) throws IOException { if ("NoSuchKey".equals(e.getS3ErrorCode())) { throw new FileNotFoundException("Key '" + key + "' does not exist in S3"); } else { - handleServiceException(e); + handleS3ServiceException(e); } } - private void handleServiceException(S3ServiceException e) throws IOException { + private void handleS3ServiceException(S3ServiceException e) throws IOException { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } else { + if(LOG.isDebugEnabled()) { + LOG.debug("S3 Error code: " + e.getS3ErrorCode() + "; S3 Error message: " + e.getS3ErrorMessage()); + } throw new S3Exception(e); } } + + private void handleServiceException(ServiceException e) throws IOException { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + else { + if(LOG.isDebugEnabled()) { + LOG.debug("Got ServiceException with Error code: " + e.getErrorCode() + ";and Error message: " + e.getErrorMessage()); + } + } + } } Modified: hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java?rev=1539259&r1=1539258&r2=1539259&view=diff ============================================================================== --- hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java (original) +++ hadoop/common/branches/branch-2.2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java Wed Nov 6 08:07:11 2013 @@ -273,7 +273,7 @@ public class NativeS3FileSystem extends setConf(conf); this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDir = - new Path("/user", System.getProperty("user.name")).makeQualified(this); + new Path("/user", System.getProperty("user.name")).makeQualified(this.uri, this.getWorkingDirectory()); } private static NativeFileSystemStore createDefaultStore(Configuration conf) { @@ -511,11 +511,11 @@ public class NativeS3FileSystem extends private FileStatus newFile(FileMetadata meta, Path path) { return new FileStatus(meta.getLength(), false, 1, getDefaultBlockSize(), - meta.getLastModified(), path.makeQualified(this)); + meta.getLastModified(), path.makeQualified(this.getUri(), this.getWorkingDirectory())); } private FileStatus newDirectory(Path path) { - return new FileStatus(0, true, 1, 0, 0, path.makeQualified(this)); + return new FileStatus(0, true, 1, 0, 0, path.makeQualified(this.getUri(), this.getWorkingDirectory())); } @Override