hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r505488 - in /lucene/hadoop/trunk: CHANGES.txt conf/hadoop-default.xml lib/jets3t-0.5.0.jar lib/jets3t.jar src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
Date Fri, 09 Feb 2007 20:49:40 GMT
Author: cutting
Date: Fri Feb  9 12:49:39 2007
New Revision: 505488

URL: http://svn.apache.org/viewvc?view=rev&rev=505488
Log:
HADOOP-882.  Upgrade to jets3t vesion 0.5, used by the S3 FileSystem.  Contributed by Michael
Stack.

Added:
    lucene/hadoop/trunk/lib/jets3t-0.5.0.jar   (with props)
Removed:
    lucene/hadoop/trunk/lib/jets3t.jar
Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/conf/hadoop-default.xml
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=505488&r1=505487&r2=505488
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Feb  9 12:49:39 2007
@@ -23,6 +23,9 @@
     The set is no longer reset with each block, but rather is now
     maintained for the life of an open file.  (Raghu Angadi via cutting)
 
+ 7. HADOOP-882.  Upgrade to jets3t version 0.5, used by the S3
+    FileSystem.  This version supports retries.  (Michael Stack via cutting)
+
 
 Branch 0.11 - unreleased
 

Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?view=diff&rev=505488&r1=505487&r2=505488
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Fri Feb  9 12:49:39 2007
@@ -355,6 +355,22 @@
   </description>
 </property>
 
+<property>
+  <name>fs.s3.block.size</name>
+  <value>1048576</value>
+  <description>
+        Block size to use writing S3.  Note, the default block size for jets3t,
+        the library at the base of the S3 filesystem, is less.  Its 131072.
+        If S3 is having a bad day requiring retries, if the retry is at some
+        point after the end of jets3t RepeatableInputStream buffer and end of
+        the S3 block, the transfer will fail. To change the jets3t block size,
+        add a jets3t.properties file to $HADOOP_HOME/conf so it can be found
+        on the CLASSPATH and amend the 's3service.stream-retry-buffer-size'
+        property. See http://jets3t.s3.amazonaws.com/toolkit/configuration.html
+        for more on jets3t configurables.
+  </description>
+</property>
+
 <!-- map/reduce properties -->
 
 <property>

Added: lucene/hadoop/trunk/lib/jets3t-0.5.0.jar
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/lib/jets3t-0.5.0.jar?view=auto&rev=505488
==============================================================================
Binary file - no diff available.

Propchange: lucene/hadoop/trunk/lib/jets3t-0.5.0.jar
------------------------------------------------------------------------------
    svn:mime-type = application/octet-stream

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java?view=diff&rev=505488&r1=505487&r2=505488
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java Fri Feb
 9 12:49:39 2007
@@ -133,7 +133,7 @@
       S3Object object = s3Service.getObject(bucket, key);
       return object.getDataInputStream();
     } catch (S3ServiceException e) {
-      if (e.getErrorCode().equals("NoSuchKey")) {
+      if (e.getS3ErrorCode().equals("NoSuchKey")) {
         return null;
       }
       if (e.getCause() instanceof IOException) {
@@ -149,7 +149,7 @@
           null, byteRangeStart, null);
       return object.getDataInputStream();
     } catch (S3ServiceException e) {
-      if (e.getErrorCode().equals("NoSuchKey")) {
+      if (e.getS3ErrorCode().equals("NoSuchKey")) {
         return null;
       }
       if (e.getCause() instanceof IOException) {



Mime
View raw message