hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r743298 - in /hadoop/core/branches/branch-0.20: CHANGES.txt src/c++/libhdfs/hdfs.c src/c++/libhdfs/hdfs.h src/c++/libhdfs/hdfs_test.c
Date Wed, 11 Feb 2009 10:59:57 GMT
Author: dhruba
Date: Wed Feb 11 10:59:57 2009
New Revision: 743298

URL: http://svn.apache.org/viewvc?rev=743298&view=rev
Log:
HADOOP-4494. Allow libhdfs to append to files.
(Pete Wyckoff via dhruba)
svn merge -c 743296 from trunk.


Modified:
    hadoop/core/branches/branch-0.20/CHANGES.txt   (contents, props changed)
    hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.c
    hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.h
    hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs_test.c

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=743298&r1=743297&r2=743298&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Wed Feb 11 10:59:57 2009
@@ -678,6 +678,9 @@
 
     HADOOP-5193. Correct calculation of edits modification time. (shv)
 
+    HADOOP-4494. Allow libhdfs to append to files.
+    (Pete Wyckoff via dhruba)
+
 Release 0.19.0 - 2008-11-18
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/core/branches/branch-0.20/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Feb 11 10:59:57 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.18/CHANGES.txt:727226
 /hadoop/core/branches/branch-0.19/CHANGES.txt:713112
-/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762
+/hadoop/core/trunk/CHANGES.txt:727001,727117,727191,727212,727228,727255,727869,728187,729052,729987,732385,732572,732777,732838,732869,733887,734870,734916,735082,736426,738602,738697,739416,740077,740157,741703,741762,743296

Modified: hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.c
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/c%2B%2B/libhdfs/hdfs.c?rev=743298&r1=743297&r2=743298&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.c (original)
+++ hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.c Wed Feb 11 10:59:57 2009
@@ -410,12 +410,17 @@
       return NULL;
     }
 
+    if ((flags & O_CREAT) && (flags & O_EXCL)) {
+      fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n");
+    }
 
     /* The hadoop java api/signature */
-    const char* method = ((flags & O_WRONLY) == 0) ? "open" : "create";
+    const char* method = ((flags & O_WRONLY) == 0) ? "open" : (flags & O_APPEND)
? "append" : "create";
     const char* signature = ((flags & O_WRONLY) == 0) ?
         JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)) :
-        JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM));
+      (flags & O_APPEND) ?
+      JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)) :
+      JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM));
 
     /* Return value */
     hdfsFile file = NULL;
@@ -459,7 +464,7 @@
         jBufferSize = jVal.i;
     }
 
-    if (flags & O_WRONLY) {
+    if ((flags & O_WRONLY) && (flags & O_APPEND) == 0) {
         //replication
 
         if (!replication) {
@@ -490,15 +495,28 @@
     /* Create and return either the FSDataInputStream or
        FSDataOutputStream references jobject jStream */
 
+    // READ?
     if ((flags & O_WRONLY) == 0) {
+      if (invokeMethod(env, &jVal, &jExc, INSTANCE, jFS, HADOOP_FS,
+                       method, signature, jPath, jBufferSize)) {
+        errno = errnoFromException(jExc, env, "org.apache.hadoop.conf."
+                                   "FileSystem::%s(%s)", method,
+                                   signature);
+        goto done;
+      }
+      // WRITE/APPEND?
+      else if ((flags & O_WRONLY) && (flags & O_APPEND)) {
         if (invokeMethod(env, &jVal, &jExc, INSTANCE, jFS, HADOOP_FS,
-                         method, signature, jPath, jBufferSize)) {
-            errno = errnoFromException(jExc, env, "org.apache.hadoop.conf."
-                                       "FileSystem::%s(%s)", method,
-                                       signature);
-            goto done;
+                         method, signature, jPath)) {
+          errno = errnoFromException(jExc, env, "org.apache.hadoop.conf."
+                                     "FileSystem::%s(%s)", method,
+                                     signature);
+          goto done;
         }
+      }
+
     }
+    // WRITE/CREATE
     else {
         jboolean jOverWrite = 1;
         if (invokeMethod(env, &jVal, &jExc, INSTANCE, jFS, HADOOP_FS,

Modified: hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.h
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/c%2B%2B/libhdfs/hdfs.h?rev=743298&r1=743297&r2=743298&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.h (original)
+++ hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs.h Wed Feb 11 10:59:57 2009
@@ -134,7 +134,8 @@
      * hdfsOpenFile - Open a hdfs file in given mode.
      * @param fs The configured filesystem handle.
      * @param path The full path to the file.
-     * @param flags Either O_RDONLY or O_WRONLY, for read-only or write-only.
+     * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY
(meaning create or overwrite i.e., implies O_TRUNCAT), 
+     * O_WRONLY|O_APPEND. Other flags are generally ignored other than (O_RDWR || (O_EXCL
& O_CREAT)) which return NULL and set errno equal ENOTSUP.
      * @param bufferSize Size of buffer for read/write - pass 0 if you want
      * to use the default configured values.
      * @param replication Block replication - pass 0 if you want to use

Modified: hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs_test.c
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/c%2B%2B/libhdfs/hdfs_test.c?rev=743298&r1=743297&r2=743298&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs_test.c (original)
+++ hadoop/core/branches/branch-0.20/src/c++/libhdfs/hdfs_test.c Wed Feb 11 10:59:57 2009
@@ -63,7 +63,7 @@
         fprintf(stderr, "Oops! Failed to connect to 'local' hdfs!\n");
         exit(-1);
     } 
- 
+
         const char* writePath = "/tmp/testfile.txt";
     {
         //Write tests
@@ -317,7 +317,75 @@
         totalResult += (result ? 0 : 1);
     }
 
+    {
+      // TEST APPENDS
+      const char *writePath = "/tmp/appends";
+
+      // CREATE
+      hdfsFile writeFile = hdfsOpenFile(fs, writePath, O_WRONLY, 0, 0, 0);
+      if(!writeFile) {
+        fprintf(stderr, "Failed to open %s for writing!\n", writePath);
+        exit(-1);
+      }
+      fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
+
+      char* buffer = "Hello,";
+      tSize num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer));
+      fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
+
+      if (hdfsFlush(fs, writeFile)) {
+        fprintf(stderr, "Failed to 'flush' %s\n", writePath); 
+        exit(-1);
+        }
+      fprintf(stderr, "Flushed %s successfully!\n", writePath); 
+
+      hdfsCloseFile(fs, writeFile);
+
+      // RE-OPEN
+      writeFile = hdfsOpenFile(fs, writePath, O_WRONLY|O_APPEND, 0, 0, 0);
+      if(!writeFile) {
+        fprintf(stderr, "Failed to open %s for writing!\n", writePath);
+        exit(-1);
+      }
+      fprintf(stderr, "Opened %s for writing successfully...\n", writePath);
 
+      buffer = " World";
+      num_written_bytes = hdfsWrite(fs, writeFile, (void*)buffer, strlen(buffer) + 1);
+      fprintf(stderr, "Wrote %d bytes\n", num_written_bytes);
+
+      if (hdfsFlush(fs, writeFile)) {
+        fprintf(stderr, "Failed to 'flush' %s\n", writePath); 
+        exit(-1);
+      }
+      fprintf(stderr, "Flushed %s successfully!\n", writePath); 
+
+      hdfsCloseFile(fs, writeFile);
+
+      // CHECK size
+      hdfsFileInfo *finfo = hdfsGetPathInfo(fs, writePath);
+      fprintf(stderr, "fileinfo->mSize: == total %s\n", ((result = (finfo->mSize ==
strlen("Hello, World") + 1)) ? "Success!" : "Failed!"));
+      totalResult += (result ? 0 : 1);
+
+      // READ and check data
+      hdfsFile readFile = hdfsOpenFile(fs, writePath, O_RDONLY, 0, 0, 0);
+      if (!readFile) {
+        fprintf(stderr, "Failed to open %s for reading!\n", writePath);
+        exit(-1);
+      }
+
+      char rdbuffer[32];
+      tSize num_read_bytes = hdfsRead(fs, readFile, (void*)rdbuffer, sizeof(rdbuffer));
+      fprintf(stderr, "Read following %d bytes:\n%s\n", 
+              num_read_bytes, rdbuffer);
+
+      fprintf(stderr, "read == Hello, World %s\n", (result = (strcmp(rdbuffer, "Hello, World")
== 0)) ? "Success!" : "Failed!");
+
+      hdfsCloseFile(fs, readFile);
+
+      // DONE test appends
+    }
+      
+      
     totalResult += (hdfsDisconnect(fs) != 0);
 
     {



Mime
View raw message