Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 10481 invoked from network); 29 Aug 2008 04:33:00 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 29 Aug 2008 04:33:00 -0000 Received: (qmail 34446 invoked by uid 500); 29 Aug 2008 04:32:58 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 34424 invoked by uid 500); 29 Aug 2008 04:32:58 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 34415 invoked by uid 99); 29 Aug 2008 04:32:58 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Aug 2008 21:32:58 -0700 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 Aug 2008 04:32:02 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 9D00E23889BA; Thu, 28 Aug 2008 21:32:02 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r690096 [1/15] - in /hadoop/core/trunk: ./ src/contrib/thriftfs/ src/contrib/thriftfs/gen-cocoa/ src/contrib/thriftfs/gen-cpp/ src/contrib/thriftfs/gen-java/ src/contrib/thriftfs/gen-java/org/ src/contrib/thriftfs/gen-java/org/apache/ src/c... Date: Fri, 29 Aug 2008 04:32:00 -0000 To: core-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20080829043202.9D00E23889BA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: omalley Date: Thu Aug 28 21:31:57 2008 New Revision: 690096 URL: http://svn.apache.org/viewvc?rev=690096&view=rev Log: HADOOP-3754. Add a thrift interface to access HDFS. (dhruba via omalley) Added: hadoop/core/trunk/src/contrib/thriftfs/ hadoop/core/trunk/src/contrib/thriftfs/README hadoop/core/trunk/src/contrib/thriftfs/build.xml hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/ hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.m hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem.cpp hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem.h hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem_server.skeleton.cpp hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_constants.cpp hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_constants.h hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_types.cpp hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_types.h hadoop/core/trunk/src/contrib/thriftfs/gen-java/ hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/ hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/ hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/ hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/ hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/BlockLocation.java hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/FileStatus.java hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/MalformedInputException.java hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/Pathname.java hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftHadoopFileSystem.java hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftHandle.java hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftIOException.java hadoop/core/trunk/src/contrib/thriftfs/gen-perl/ hadoop/core/trunk/src/contrib/thriftfs/gen-perl/Constants.pm hadoop/core/trunk/src/contrib/thriftfs/gen-perl/ThriftHadoopFileSystem.pm hadoop/core/trunk/src/contrib/thriftfs/gen-perl/Types.pm hadoop/core/trunk/src/contrib/thriftfs/gen-php/ hadoop/core/trunk/src/contrib/thriftfs/gen-php/ThriftHadoopFileSystem.php hadoop/core/trunk/src/contrib/thriftfs/gen-php/hadoopfs_types.php hadoop/core/trunk/src/contrib/thriftfs/gen-py/ hadoop/core/trunk/src/contrib/thriftfs/gen-py/__init__.py hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem-remote (with props) hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem.py hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem.pyc (with props) hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/__init__.py hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/__init__.pyc (with props) hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/constants.py hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ttypes.py hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ttypes.pyc (with props) hadoop/core/trunk/src/contrib/thriftfs/gen-rb/ hadoop/core/trunk/src/contrib/thriftfs/gen-rb/ThriftHadoopFileSystem.rb hadoop/core/trunk/src/contrib/thriftfs/gen-rb/hadoopfs_constants.rb hadoop/core/trunk/src/contrib/thriftfs/gen-rb/hadoopfs_types.rb hadoop/core/trunk/src/contrib/thriftfs/gen-st/ hadoop/core/trunk/src/contrib/thriftfs/gen-st/hadoopfs.st hadoop/core/trunk/src/contrib/thriftfs/if/ hadoop/core/trunk/src/contrib/thriftfs/if/hadoopfs.thrift hadoop/core/trunk/src/contrib/thriftfs/lib/ hadoop/core/trunk/src/contrib/thriftfs/lib/README hadoop/core/trunk/src/contrib/thriftfs/lib/Thrift.LICENSE hadoop/core/trunk/src/contrib/thriftfs/lib/hadoopthriftapi.jar (with props) hadoop/core/trunk/src/contrib/thriftfs/lib/libthrift.jar (with props) hadoop/core/trunk/src/contrib/thriftfs/scripts/ hadoop/core/trunk/src/contrib/thriftfs/scripts/hdfs.py (with props) hadoop/core/trunk/src/contrib/thriftfs/scripts/start_thrift_server.sh (with props) hadoop/core/trunk/src/contrib/thriftfs/src/ hadoop/core/trunk/src/contrib/thriftfs/src/java/ hadoop/core/trunk/src/contrib/thriftfs/src/java/org/ hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/ hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/ hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/thriftfs/ hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/thriftfs/HadoopThriftServer.java hadoop/core/trunk/src/contrib/thriftfs/test/ hadoop/core/trunk/src/contrib/thriftfs/test/org/ hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/ hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/ hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/thriftfs/ hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/thriftfs/TestThriftfs.java Modified: hadoop/core/trunk/CHANGES.txt Modified: hadoop/core/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=690096&r1=690095&r2=690096&view=diff ============================================================================== --- hadoop/core/trunk/CHANGES.txt (original) +++ hadoop/core/trunk/CHANGES.txt Thu Aug 28 21:31:57 2008 @@ -106,6 +106,8 @@ HADOOP-3746. Add a fair share scheduler. (Matei Zaharia via omalley) + HADOOP-3754. Add a thrift interface to access HDFS. (dhruba via omalley) + IMPROVEMENTS HADOOP-3908. Fuse-dfs: better error message if llibhdfs.so doesn't exist. Added: hadoop/core/trunk/src/contrib/thriftfs/README URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/README?rev=690096&view=auto ============================================================================== --- hadoop/core/trunk/src/contrib/thriftfs/README (added) +++ hadoop/core/trunk/src/contrib/thriftfs/README Thu Aug 28 21:31:57 2008 @@ -0,0 +1,39 @@ + Thrift API for HDFS + ================== + +Introduction: +============ + +The Hadoop Distributed File System is written in Java. An application +that wants to store/fetch data to/from HDFS can use the Java API +This means that applications that are not written in Java cannot +access HDFS in an elegant manner. + +Thrift is a software framework for scalable cross-language services +development. It combines a powerful software stack with a code generation +engine to build services that work efficiently and seamlessly +between C++, Java, Python, PHP, and Ruby. + +This project exposes HDFS APIs using the Thrift software stack. This +allows applciations written in a myriad of languages to access +HDFS elegantly. + + +The Application Programming Interface (API) +=========================================== +The HDFS API that is exposed through Thrift can be found in if/hadoopfs.thrift. + +Compilation +=========== +The compilation process creates a server org.apache.hadoop.thriftfs.HadooopThriftServer +that implements the Thrift interface defined in if/hadoopfs.thrift. + +Th thrift compiler is used to generate API stubs in python, php, ruby, +cocoa, etc. The generated code is checked into the directories gen-*. +The generated java API is checked into lib/hadoopthriftapi.jar. + +There is a sample python script hdfs.py in the scripts directory. This python +script, when invoked, creates a HadoopThriftServer in the background, and then +communicates wth HDFS using the API. This script is for demonstration purposes +only. + Added: hadoop/core/trunk/src/contrib/thriftfs/build.xml URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/build.xml?rev=690096&view=auto ============================================================================== --- hadoop/core/trunk/src/contrib/thriftfs/build.xml (added) +++ hadoop/core/trunk/src/contrib/thriftfs/build.xml Thu Aug 28 21:31:57 2008 @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Added: hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h?rev=690096&view=auto ============================================================================== --- hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h (added) +++ hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h Thu Aug 28 21:31:57 2008 @@ -0,0 +1,214 @@ +/** + * Autogenerated by Thrift + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + */ + +#import + +#import +#import +#import + + +@interface ThriftHandle : NSObject { + int64_t __id; + + BOOL __id_isset; +} + +- (id) initWithId: (int64_t) id; + +- (void) read: (id ) inProtocol; +- (void) write: (id ) outProtocol; + +- (int64_t) id; +- (void) setId: (int64_t) id; +- (BOOL) idIsSet; + +@end + +@interface Pathname : NSObject { + NSString * __pathname; + + BOOL __pathname_isset; +} + +- (id) initWithPathname: (NSString *) pathname; + +- (void) read: (id ) inProtocol; +- (void) write: (id ) outProtocol; + +- (NSString *) pathname; +- (void) setPathname: (NSString *) pathname; +- (BOOL) pathnameIsSet; + +@end + +@interface FileStatus : NSObject { + NSString * __path; + int64_t __length; + BOOL __isdir; + int16_t __block_replication; + int64_t __blocksize; + int64_t __modification_time; + NSString * __permission; + NSString * __owner; + NSString * __group; + + BOOL __path_isset; + BOOL __length_isset; + BOOL __isdir_isset; + BOOL __block_replication_isset; + BOOL __blocksize_isset; + BOOL __modification_time_isset; + BOOL __permission_isset; + BOOL __owner_isset; + BOOL __group_isset; +} + +- (id) initWithPath: (NSString *) path length: (int64_t) length isdir: (BOOL) isdir block_replication: (int16_t) block_replication blocksize: (int64_t) blocksize modification_time: (int64_t) modification_time permission: (NSString *) permission owner: (NSString *) owner group: (NSString *) group; + +- (void) read: (id ) inProtocol; +- (void) write: (id ) outProtocol; + +- (NSString *) path; +- (void) setPath: (NSString *) path; +- (BOOL) pathIsSet; + +- (int64_t) length; +- (void) setLength: (int64_t) length; +- (BOOL) lengthIsSet; + +- (BOOL) isdir; +- (void) setIsdir: (BOOL) isdir; +- (BOOL) isdirIsSet; + +- (int16_t) block_replication; +- (void) setBlock_replication: (int16_t) block_replication; +- (BOOL) block_replicationIsSet; + +- (int64_t) blocksize; +- (void) setBlocksize: (int64_t) blocksize; +- (BOOL) blocksizeIsSet; + +- (int64_t) modification_time; +- (void) setModification_time: (int64_t) modification_time; +- (BOOL) modification_timeIsSet; + +- (NSString *) permission; +- (void) setPermission: (NSString *) permission; +- (BOOL) permissionIsSet; + +- (NSString *) owner; +- (void) setOwner: (NSString *) owner; +- (BOOL) ownerIsSet; + +- (NSString *) group; +- (void) setGroup: (NSString *) group; +- (BOOL) groupIsSet; + +@end + +@interface BlockLocation : NSObject { + NSArray * __hosts; + NSArray * __names; + int64_t __offset; + int64_t __length; + + BOOL __hosts_isset; + BOOL __names_isset; + BOOL __offset_isset; + BOOL __length_isset; +} + +- (id) initWithHosts: (NSArray *) hosts names: (NSArray *) names offset: (int64_t) offset length: (int64_t) length; + +- (void) read: (id ) inProtocol; +- (void) write: (id ) outProtocol; + +- (NSArray *) hosts; +- (void) setHosts: (NSArray *) hosts; +- (BOOL) hostsIsSet; + +- (NSArray *) names; +- (void) setNames: (NSArray *) names; +- (BOOL) namesIsSet; + +- (int64_t) offset; +- (void) setOffset: (int64_t) offset; +- (BOOL) offsetIsSet; + +- (int64_t) length; +- (void) setLength: (int64_t) length; +- (BOOL) lengthIsSet; + +@end + +@interface MalformedInputException : NSException { + NSString * __message; + + BOOL __message_isset; +} + +- (id) initWithMessage: (NSString *) message; + +- (void) read: (id ) inProtocol; +- (void) write: (id ) outProtocol; + +- (NSString *) message; +- (void) setMessage: (NSString *) message; +- (BOOL) messageIsSet; + +@end + +@interface ThriftIOException : NSException { + NSString * __message; + + BOOL __message_isset; +} + +- (id) initWithMessage: (NSString *) message; + +- (void) read: (id ) inProtocol; +- (void) write: (id ) outProtocol; + +- (NSString *) message; +- (void) setMessage: (NSString *) message; +- (BOOL) messageIsSet; + +@end + +@protocol ThriftHadoopFileSystem +- (void) setInactivityTimeoutPeriod: (int64_t) periodInSeconds; // throws TException +- (void) shutdown: (int32_t) status; // throws TException +- (ThriftHandle *) create: (Pathname *) path; // throws ThriftIOException *, TException +- (ThriftHandle *) createFile: (Pathname *) path : (int16_t) mode : (BOOL) overwrite : (int32_t) bufferSize : (int16_t) block_replication : (int64_t) blocksize; // throws ThriftIOException *, TException +- (ThriftHandle *) open: (Pathname *) path; // throws ThriftIOException *, TException +- (ThriftHandle *) append: (Pathname *) path; // throws ThriftIOException *, TException +- (BOOL) write: (ThriftHandle *) handle : (NSString *) data; // throws ThriftIOException *, TException +- (NSString *) read: (ThriftHandle *) handle : (int64_t) offset : (int32_t) size; // throws ThriftIOException *, TException +- (BOOL) close: (ThriftHandle *) out; // throws ThriftIOException *, TException +- (BOOL) rm: (Pathname *) path : (BOOL) recursive; // throws ThriftIOException *, TException +- (BOOL) rename: (Pathname *) path : (Pathname *) dest; // throws ThriftIOException *, TException +- (BOOL) mkdirs: (Pathname *) path; // throws ThriftIOException *, TException +- (BOOL) exists: (Pathname *) path; // throws ThriftIOException *, TException +- (FileStatus *) stat: (Pathname *) path; // throws ThriftIOException *, TException +- (NSArray *) listStatus: (Pathname *) path; // throws ThriftIOException *, TException +- (void) chmod: (Pathname *) path : (int16_t) mode; // throws ThriftIOException *, TException +- (void) chown: (Pathname *) path : (NSString *) owner : (NSString *) group; // throws ThriftIOException *, TException +- (void) setReplication: (Pathname *) path : (int16_t) replication; // throws ThriftIOException *, TException +- (NSArray *) getFileBlockLocations: (Pathname *) path : (int64_t) start : (int64_t) length; // throws ThriftIOException *, TException +@end + +@interface ThriftHadoopFileSystemClient : NSObject { + id inProtocol; + id outProtocol; +} +- (id) initWithProtocol: (id ) protocol; +- (id) initWithInProtocol: (id ) inProtocol outProtocol: (id ) outProtocol; +@end + +@interface hadoopfsConstants { +} +@end