hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r603000 - in /lucene/hadoop/trunk: CHANGES.txt src/c++/libhdfs/hdfs.c
Date Mon, 10 Dec 2007 19:14:13 GMT
Author: dhruba
Date: Mon Dec 10 11:14:12 2007
New Revision: 603000

URL: http://svn.apache.org/viewvc?rev=603000&view=rev
Log:
HADOOP-2158. hdfsListDirectory calls FileSystem.listStatus instead
of FileSystem.listPaths. This reduces the number of RPC calls on the
namenode, thereby improving scalability.  (Christian Kunz via dhruba)


Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/c++/libhdfs/hdfs.c

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=603000&r1=602999&r2=603000&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Dec 10 11:14:12 2007
@@ -120,6 +120,10 @@
     HADOOP-1900. Makes the heartbeat and task event queries interval 
     dependent on the cluster size.  (Amareshwari Sri Ramadasu via ddas)
 
+    HADOOP-2158. hdfsListDirectory calls FileSystem.listStatus instead
+    of FileSystem.listPaths. This reduces the number of RPC calls on the
+    namenode, thereby improving scalability.  (Christian Kunz via dhruba)
+
   BUG FIXES
 
     HADOOP-2100.  Remove faulty check for existence of $HADOOP_PID_DIR and let

Modified: lucene/hadoop/trunk/src/c++/libhdfs/hdfs.c
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/libhdfs/hdfs.c?rev=603000&r1=602999&r2=603000&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/c++/libhdfs/hdfs.c (original)
+++ lucene/hadoop/trunk/src/c++/libhdfs/hdfs.c Mon Dec 10 11:14:12 2007
@@ -1316,41 +1316,9 @@
 
  
 static int
-getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo *fileInfo)
+getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo)
 {
-    // JAVA EQUIVALENT:
-    //  fs.isDirectory(f)
-    //  fs.lastModified() ??
-    //  fs.getLength(f)
-    //  f.getPath()
-
-    jobject jStat;
-    jvalue  jVal;
-
-    if (invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS,
-                     "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"),
-                     jPath) != 0) {
-        fprintf(stderr, "Call to org.apache.hadoop.fs."
-                "FileSystem::exists failed!\n");
-        errno = EINTERNAL;
-        return -1;
-    }
-
-    if (jVal.z == 0) {
-      errno = ENOENT;
-      return -1;
-    }
-
-    if (invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS,
-                     "getFileStatus", JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)),
-                     jPath) != 0) {
-        fprintf(stderr, "Call to org.apache.hadoop.fs."
-                "FileSystem::getFileStatus failed!\n");
-        errno = EINTERNAL;
-        return -1;
-    }
-    jStat = jVal.l;
-
+    jvalue jVal;
     if (invokeMethod(env, &jVal, INSTANCE, jStat,
                      HADOOP_STAT, "isDir", "()Z") != 0) {
         fprintf(stderr, "Call to org.apache.hadoop.fs."
@@ -1398,6 +1366,17 @@
         fileInfo->mSize = jVal.j;
     }
 
+    jobject jPath;
+    if (invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT,
+                     "getPath", "()Lorg/apache/hadoop/fs/Path;") ||
+            jVal.l == NULL) { 
+        fprintf(stderr, "Call to org.apache.hadoop.fs."
+                "FileStatus::getPath failed!\n");
+        errno = EINTERNAL;
+        return -1;
+    }
+    jPath = jVal.l;
+
     jstring     jPathName;
     const char *cPathName;
     if (invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH,
@@ -1405,17 +1384,59 @@
         fprintf(stderr, "Call to org.apache.hadoop.fs."
                 "Path::toString failed!\n");
         errno = EINTERNAL;
+        destroyLocalReference(env, jPath);
         return -1;
     }
     jPathName = jVal.l;
     cPathName = (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL));
     fileInfo->mName = strdup(cPathName);
     (*env)->ReleaseStringUTFChars(env, jPathName, cPathName);
+    destroyLocalReference(env, jPath);
     destroyLocalReference(env, jPathName);
 
     return 0;
 }
 
+static int
+getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo *fileInfo)
+{
+    // JAVA EQUIVALENT:
+    //  fs.isDirectory(f)
+    //  fs.lastModified() ??
+    //  fs.getLength(f)
+    //  f.getPath()
+
+    jobject jStat;
+    jvalue  jVal;
+
+    if (invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS,
+                     "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"),
+                     jPath) != 0) {
+        fprintf(stderr, "Call to org.apache.hadoop.fs."
+                "FileSystem::exists failed!\n");
+        errno = EINTERNAL;
+        return -1;
+    }
+
+    if (jVal.z == 0) {
+      errno = ENOENT;
+      return -1;
+    }
+
+    if (invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS,
+                     "getFileStatus", JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)),
+                     jPath) != 0) {
+        fprintf(stderr, "Call to org.apache.hadoop.fs."
+                "FileSystem::getFileStatus failed!\n");
+        errno = EINTERNAL;
+        return -1;
+    }
+    jStat = jVal.l;
+    int ret =  getFileInfoFromStat(env, jStat, fileInfo); 
+    destroyLocalReference(env, jStat);
+    return ret;
+}
+
 
 
 hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries)
@@ -1441,11 +1462,11 @@
 
     jobjectArray jPathList = NULL;
     jvalue jVal;
-    if (invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "listPaths",
-                     JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_PATH)),
+    if (invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus",
+                     JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)),
                      jPath) != 0) {
         fprintf(stderr, "Call to org.apache.hadoop.fs."
-                "FileSystem::listPaths failed!\n");
+                "FileSystem::listStatus failed!\n");
         errno = EINTERNAL;
         destroyLocalReference(env, jPath);
         return NULL;
@@ -1469,17 +1490,17 @@
 
     //Save path information in pathList
     jsize i;
-    jobject tmpPath;
+    jobject tmpStat;
     for (i=0; i < jPathListSize; ++i) {
-        tmpPath = (*env)->GetObjectArrayElement(env, jPathList, i);
-        if (getFileInfo(env, jFS, tmpPath, &pathList[i])) {
+        tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i);
+        if (getFileInfoFromStat(env, tmpStat, &pathList[i])) {
             errno = EINTERNAL;
             hdfsFreeFileInfo(pathList, jPathListSize);
-            destroyLocalReference(env, tmpPath);
+            destroyLocalReference(env, tmpStat);
             pathList = NULL;
             goto done;
         }
-        destroyLocalReference(env, tmpPath);
+        destroyLocalReference(env, tmpStat);
     }
 
     done:



Mime
View raw message