hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r690096 [1/15] - in /hadoop/core/trunk: ./ src/contrib/thriftfs/ src/contrib/thriftfs/gen-cocoa/ src/contrib/thriftfs/gen-cpp/ src/contrib/thriftfs/gen-java/ src/contrib/thriftfs/gen-java/org/ src/contrib/thriftfs/gen-java/org/apache/ src/c...
Date Fri, 29 Aug 2008 04:32:00 GMT
Author: omalley
Date: Thu Aug 28 21:31:57 2008
New Revision: 690096

URL: http://svn.apache.org/viewvc?rev=690096&view=rev
Log:
HADOOP-3754. Add a thrift interface to access HDFS. (dhruba via omalley)

Added:
    hadoop/core/trunk/src/contrib/thriftfs/
    hadoop/core/trunk/src/contrib/thriftfs/README
    hadoop/core/trunk/src/contrib/thriftfs/build.xml
    hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/
    hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h
    hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.m
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem.cpp
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem.h
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/ThriftHadoopFileSystem_server.skeleton.cpp
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_constants.cpp
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_constants.h
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_types.cpp
    hadoop/core/trunk/src/contrib/thriftfs/gen-cpp/hadoopfs_types.h
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/BlockLocation.java
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/FileStatus.java
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/MalformedInputException.java
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/Pathname.java
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftHadoopFileSystem.java
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftHandle.java
    hadoop/core/trunk/src/contrib/thriftfs/gen-java/org/apache/hadoop/thriftfs/api/ThriftIOException.java
    hadoop/core/trunk/src/contrib/thriftfs/gen-perl/
    hadoop/core/trunk/src/contrib/thriftfs/gen-perl/Constants.pm
    hadoop/core/trunk/src/contrib/thriftfs/gen-perl/ThriftHadoopFileSystem.pm
    hadoop/core/trunk/src/contrib/thriftfs/gen-perl/Types.pm
    hadoop/core/trunk/src/contrib/thriftfs/gen-php/
    hadoop/core/trunk/src/contrib/thriftfs/gen-php/ThriftHadoopFileSystem.php
    hadoop/core/trunk/src/contrib/thriftfs/gen-php/hadoopfs_types.php
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/__init__.py
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem-remote 
 (with props)
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem.py
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ThriftHadoopFileSystem.pyc   (with
props)
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/__init__.py
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/__init__.pyc   (with props)
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/constants.py
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ttypes.py
    hadoop/core/trunk/src/contrib/thriftfs/gen-py/hadoopfs/ttypes.pyc   (with props)
    hadoop/core/trunk/src/contrib/thriftfs/gen-rb/
    hadoop/core/trunk/src/contrib/thriftfs/gen-rb/ThriftHadoopFileSystem.rb
    hadoop/core/trunk/src/contrib/thriftfs/gen-rb/hadoopfs_constants.rb
    hadoop/core/trunk/src/contrib/thriftfs/gen-rb/hadoopfs_types.rb
    hadoop/core/trunk/src/contrib/thriftfs/gen-st/
    hadoop/core/trunk/src/contrib/thriftfs/gen-st/hadoopfs.st
    hadoop/core/trunk/src/contrib/thriftfs/if/
    hadoop/core/trunk/src/contrib/thriftfs/if/hadoopfs.thrift
    hadoop/core/trunk/src/contrib/thriftfs/lib/
    hadoop/core/trunk/src/contrib/thriftfs/lib/README
    hadoop/core/trunk/src/contrib/thriftfs/lib/Thrift.LICENSE
    hadoop/core/trunk/src/contrib/thriftfs/lib/hadoopthriftapi.jar   (with props)
    hadoop/core/trunk/src/contrib/thriftfs/lib/libthrift.jar   (with props)
    hadoop/core/trunk/src/contrib/thriftfs/scripts/
    hadoop/core/trunk/src/contrib/thriftfs/scripts/hdfs.py   (with props)
    hadoop/core/trunk/src/contrib/thriftfs/scripts/start_thrift_server.sh   (with props)
    hadoop/core/trunk/src/contrib/thriftfs/src/
    hadoop/core/trunk/src/contrib/thriftfs/src/java/
    hadoop/core/trunk/src/contrib/thriftfs/src/java/org/
    hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/
    hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/
    hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/thriftfs/
    hadoop/core/trunk/src/contrib/thriftfs/src/java/org/apache/hadoop/thriftfs/HadoopThriftServer.java
    hadoop/core/trunk/src/contrib/thriftfs/test/
    hadoop/core/trunk/src/contrib/thriftfs/test/org/
    hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/
    hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/
    hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/thriftfs/
    hadoop/core/trunk/src/contrib/thriftfs/test/org/apache/hadoop/thriftfs/TestThriftfs.java
Modified:
    hadoop/core/trunk/CHANGES.txt

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=690096&r1=690095&r2=690096&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Aug 28 21:31:57 2008
@@ -106,6 +106,8 @@
 
     HADOOP-3746. Add a fair share scheduler. (Matei Zaharia via omalley)
 
+    HADOOP-3754. Add a thrift interface to access HDFS. (dhruba via omalley)
+
   IMPROVEMENTS
 
     HADOOP-3908. Fuse-dfs: better error message if llibhdfs.so doesn't exist.

Added: hadoop/core/trunk/src/contrib/thriftfs/README
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/README?rev=690096&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/thriftfs/README (added)
+++ hadoop/core/trunk/src/contrib/thriftfs/README Thu Aug 28 21:31:57 2008
@@ -0,0 +1,39 @@
+                        Thrift API for HDFS
+                        ==================
+
+Introduction:
+============
+
+The Hadoop Distributed File System is written in Java. An application
+that wants to store/fetch data to/from HDFS can use the Java API
+This means that applications that are not written in Java cannot
+access HDFS in an elegant manner.
+
+Thrift is a software framework for scalable cross-language services 
+development. It combines a powerful software stack with a code generation 
+engine to build services that work efficiently and seamlessly 
+between C++, Java, Python, PHP, and Ruby.
+
+This project exposes HDFS APIs using the Thrift software stack. This
+allows applciations written in a myriad of languages to access
+HDFS elegantly.
+
+
+The Application Programming Interface (API)
+===========================================
+The HDFS API that is exposed through Thrift can be found in if/hadoopfs.thrift.
+
+Compilation
+===========
+The compilation process creates a server org.apache.hadoop.thriftfs.HadooopThriftServer
+that implements the Thrift interface defined in if/hadoopfs.thrift.
+
+Th thrift compiler is used to generate API stubs in python, php, ruby,
+cocoa, etc. The generated code is checked into the directories gen-*.
+The generated java API is checked into lib/hadoopthriftapi.jar.
+
+There is a sample python script hdfs.py in the scripts directory. This python 
+script, when invoked, creates a HadoopThriftServer in the background, and then
+communicates wth HDFS using the API. This script is for demonstration purposes
+only.
+

Added: hadoop/core/trunk/src/contrib/thriftfs/build.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/build.xml?rev=690096&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/thriftfs/build.xml (added)
+++ hadoop/core/trunk/src/contrib/thriftfs/build.xml Thu Aug 28 21:31:57 2008
@@ -0,0 +1,64 @@
+<?xml version="1.0"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!-- 
+Before you can run these subtargets directly, you need 
+to call at top-level: ant deploy-contrib compile-core-test
+-->
+<project name="thriftfs" default="jar">
+
+  <import file="../build-contrib.xml"/>
+
+  <!-- create the list of files to add to the classpath -->
+  <fileset dir="${hadoop.root}/lib" id="class.path">
+    <include name="**/*.jar" />
+    <exclude name="**/excluded/" />
+  </fileset>
+
+  <!-- Override jar target to specify main class -->
+  <target name="jar" depends="compile">
+    <jar
+      jarfile="${build.dir}/hadoop-${version}-${name}.jar"
+      basedir="${build.classes}"      
+    >
+  	<manifest>
+	    <attribute name="Main-Class" value="org.apache.hadoop.thriftfs.HadooopThriftServer"/>
+	</manifest>
+    </jar>
+  </target>
+
+  <!-- Run only pure-Java unit tests. superdottest -->
+  <target name="test">
+   <antcall target="hadoopbuildcontrib.test"> 
+     <param name="test.exclude" value="TestStreamedMerge"/>
+   </antcall>
+  </target>  
+ 
+  <!-- Run all unit tests
+  This is not called as part of the nightly build
+  because it will only run on platforms that have standard 
+  Unix utilities available. 
+  -->
+ <target name="test-unix">
+   <antcall target="hadoopbuildcontrib.test">
+   </antcall>
+ </target>  
+
+
+</project>

Added: hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h?rev=690096&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h (added)
+++ hadoop/core/trunk/src/contrib/thriftfs/gen-cocoa/hadoopfs.h Thu Aug 28 21:31:57 2008
@@ -0,0 +1,214 @@
+/**
+ * Autogenerated by Thrift
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+
+#import <Cocoa/Cocoa.h>
+
+#import <TProtocol.h>
+#import <TApplicationException.h>
+#import <TProtocolUtil.h>
+
+
+@interface ThriftHandle : NSObject {
+  int64_t __id;
+
+  BOOL __id_isset;
+}
+
+- (id) initWithId: (int64_t) id;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (int64_t) id;
+- (void) setId: (int64_t) id;
+- (BOOL) idIsSet;
+
+@end
+
+@interface Pathname : NSObject {
+  NSString * __pathname;
+
+  BOOL __pathname_isset;
+}
+
+- (id) initWithPathname: (NSString *) pathname;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) pathname;
+- (void) setPathname: (NSString *) pathname;
+- (BOOL) pathnameIsSet;
+
+@end
+
+@interface FileStatus : NSObject {
+  NSString * __path;
+  int64_t __length;
+  BOOL __isdir;
+  int16_t __block_replication;
+  int64_t __blocksize;
+  int64_t __modification_time;
+  NSString * __permission;
+  NSString * __owner;
+  NSString * __group;
+
+  BOOL __path_isset;
+  BOOL __length_isset;
+  BOOL __isdir_isset;
+  BOOL __block_replication_isset;
+  BOOL __blocksize_isset;
+  BOOL __modification_time_isset;
+  BOOL __permission_isset;
+  BOOL __owner_isset;
+  BOOL __group_isset;
+}
+
+- (id) initWithPath: (NSString *) path length: (int64_t) length isdir: (BOOL) isdir block_replication:
(int16_t) block_replication blocksize: (int64_t) blocksize modification_time: (int64_t) modification_time
permission: (NSString *) permission owner: (NSString *) owner group: (NSString *) group;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) path;
+- (void) setPath: (NSString *) path;
+- (BOOL) pathIsSet;
+
+- (int64_t) length;
+- (void) setLength: (int64_t) length;
+- (BOOL) lengthIsSet;
+
+- (BOOL) isdir;
+- (void) setIsdir: (BOOL) isdir;
+- (BOOL) isdirIsSet;
+
+- (int16_t) block_replication;
+- (void) setBlock_replication: (int16_t) block_replication;
+- (BOOL) block_replicationIsSet;
+
+- (int64_t) blocksize;
+- (void) setBlocksize: (int64_t) blocksize;
+- (BOOL) blocksizeIsSet;
+
+- (int64_t) modification_time;
+- (void) setModification_time: (int64_t) modification_time;
+- (BOOL) modification_timeIsSet;
+
+- (NSString *) permission;
+- (void) setPermission: (NSString *) permission;
+- (BOOL) permissionIsSet;
+
+- (NSString *) owner;
+- (void) setOwner: (NSString *) owner;
+- (BOOL) ownerIsSet;
+
+- (NSString *) group;
+- (void) setGroup: (NSString *) group;
+- (BOOL) groupIsSet;
+
+@end
+
+@interface BlockLocation : NSObject {
+  NSArray * __hosts;
+  NSArray * __names;
+  int64_t __offset;
+  int64_t __length;
+
+  BOOL __hosts_isset;
+  BOOL __names_isset;
+  BOOL __offset_isset;
+  BOOL __length_isset;
+}
+
+- (id) initWithHosts: (NSArray *) hosts names: (NSArray *) names offset: (int64_t) offset
length: (int64_t) length;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSArray *) hosts;
+- (void) setHosts: (NSArray *) hosts;
+- (BOOL) hostsIsSet;
+
+- (NSArray *) names;
+- (void) setNames: (NSArray *) names;
+- (BOOL) namesIsSet;
+
+- (int64_t) offset;
+- (void) setOffset: (int64_t) offset;
+- (BOOL) offsetIsSet;
+
+- (int64_t) length;
+- (void) setLength: (int64_t) length;
+- (BOOL) lengthIsSet;
+
+@end
+
+@interface MalformedInputException : NSException {
+  NSString * __message;
+
+  BOOL __message_isset;
+}
+
+- (id) initWithMessage: (NSString *) message;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) message;
+- (void) setMessage: (NSString *) message;
+- (BOOL) messageIsSet;
+
+@end
+
+@interface ThriftIOException : NSException {
+  NSString * __message;
+
+  BOOL __message_isset;
+}
+
+- (id) initWithMessage: (NSString *) message;
+
+- (void) read: (id <TProtocol>) inProtocol;
+- (void) write: (id <TProtocol>) outProtocol;
+
+- (NSString *) message;
+- (void) setMessage: (NSString *) message;
+- (BOOL) messageIsSet;
+
+@end
+
+@protocol ThriftHadoopFileSystem <NSObject>
+- (void) setInactivityTimeoutPeriod: (int64_t) periodInSeconds;  // throws TException
+- (void) shutdown: (int32_t) status;  // throws TException
+- (ThriftHandle *) create: (Pathname *) path;  // throws ThriftIOException *, TException
+- (ThriftHandle *) createFile: (Pathname *) path : (int16_t) mode : (BOOL) overwrite : (int32_t)
bufferSize : (int16_t) block_replication : (int64_t) blocksize;  // throws ThriftIOException
*, TException
+- (ThriftHandle *) open: (Pathname *) path;  // throws ThriftIOException *, TException
+- (ThriftHandle *) append: (Pathname *) path;  // throws ThriftIOException *, TException
+- (BOOL) write: (ThriftHandle *) handle : (NSString *) data;  // throws ThriftIOException
*, TException
+- (NSString *) read: (ThriftHandle *) handle : (int64_t) offset : (int32_t) size;  // throws
ThriftIOException *, TException
+- (BOOL) close: (ThriftHandle *) out;  // throws ThriftIOException *, TException
+- (BOOL) rm: (Pathname *) path : (BOOL) recursive;  // throws ThriftIOException *, TException
+- (BOOL) rename: (Pathname *) path : (Pathname *) dest;  // throws ThriftIOException *, TException
+- (BOOL) mkdirs: (Pathname *) path;  // throws ThriftIOException *, TException
+- (BOOL) exists: (Pathname *) path;  // throws ThriftIOException *, TException
+- (FileStatus *) stat: (Pathname *) path;  // throws ThriftIOException *, TException
+- (NSArray *) listStatus: (Pathname *) path;  // throws ThriftIOException *, TException
+- (void) chmod: (Pathname *) path : (int16_t) mode;  // throws ThriftIOException *, TException
+- (void) chown: (Pathname *) path : (NSString *) owner : (NSString *) group;  // throws ThriftIOException
*, TException
+- (void) setReplication: (Pathname *) path : (int16_t) replication;  // throws ThriftIOException
*, TException
+- (NSArray *) getFileBlockLocations: (Pathname *) path : (int64_t) start : (int64_t) length;
 // throws ThriftIOException *, TException
+@end
+
+@interface ThriftHadoopFileSystemClient : NSObject <ThriftHadoopFileSystem> {
+  id <TProtocol> inProtocol;
+  id <TProtocol> outProtocol;
+}
+- (id) initWithProtocol: (id <TProtocol>) protocol;
+- (id) initWithInProtocol: (id <TProtocol>) inProtocol outProtocol: (id <TProtocol>)
outProtocol;
+@end
+
+@interface hadoopfsConstants {
+}
+@end



Mime
View raw message