hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1361440 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/ src/main/native/
Date Fri, 13 Jul 2012 23:28:04 GMT
Author: eli
Date: Fri Jul 13 23:28:03 2012
New Revision: 1361440

URL: http://svn.apache.org/viewvc?rev=1361440&view=rev
Log:
HDFS-3606. libhdfs: create self-contained unit test. Contributed by Colin Patrick McCabe

Added:
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/expect.h
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.c
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.h
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_libhdfs_threaded.c
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_native_mini_dfs.c
Modified:
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/pom.xml
    hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt

Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1361440&r1=1361439&r2=1361440&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Jul 13 23:28:03 2012
@@ -319,6 +319,9 @@ Branch-2 ( Unreleased changes )
     HDFS-3612. Single namenode image directory config warning can
     be improved. (Andy Isaacson via harsh)
 
+    HDFS-3606. libhdfs: create self-contained unit test.
+    (Colin Patrick McCabe via eli)
+
   OPTIMIZATIONS
 
     HDFS-2982. Startup performance suffers when there are many edit log

Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1361440&r1=1361439&r2=1361440&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/pom.xml Fri Jul 13 23:28:03 2012
@@ -422,18 +422,23 @@ http://maven.apache.org/xsd/maven-4.0.0.
                   </target>
                 </configuration>
               </execution>
-              <!-- TODO wire here native testcases
               <execution>
-                <id>test</id>
+                <id>native_tests</id>
                 <phase>test</phase>
-                <goals>
-                  <goal>test</goal>
-                </goals>
+                <goals><goal>run</goal></goals>
                 <configuration>
-                  <destDir>${project.build.directory}/native/target</destDir>
+                  <target>
+                    <property name="compile_classpath" refid="maven.compile.classpath"/>
+                    <property name="test_classpath" refid="maven.test.classpath"/>
+                    <exec executable="${project.build.directory}/native/test_libhdfs_threaded"
dir="${project.build.directory}/native/" failonerror="true">
+                      <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
+                    </exec>
+                    <exec executable="${project.build.directory}/native/test_native_mini_dfs"
dir="${project.build.directory}/native/" failonerror="true">
+                      <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
+                    </exec>
+                  </target>
                 </configuration>
               </execution>
-              -->
             </executions>
           </plugin>
         </plugins>

Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt?rev=1361440&r1=1361439&r2=1361440&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt Fri Jul 13 23:28:03
2012
@@ -132,4 +132,28 @@ target_link_libraries(hdfs_write
 )
 output_directory(hdfs_write target/usr/local/bin)
 
+add_library(native_mini_dfs
+    main/native/native_mini_dfs.c
+)
+target_link_libraries(native_mini_dfs
+    hdfs
+)
+
+add_executable(test_native_mini_dfs
+    main/native/test_native_mini_dfs.c
+)
+target_link_libraries(test_native_mini_dfs
+    native_mini_dfs
+)
+
+add_executable(test_libhdfs_threaded
+    main/native/test_libhdfs_threaded.c
+)
+target_link_libraries(test_libhdfs_threaded
+    hdfs
+    native_mini_dfs
+    pthread
+)
+
 add_subdirectory(contrib/fuse-dfs/src)
+

Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/expect.h
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/expect.h?rev=1361440&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/expect.h (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/expect.h Fri Jul 13
23:28:03 2012
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_NATIVE_TESTS_EXPECT_H
+#define LIBHDFS_NATIVE_TESTS_EXPECT_H
+
+#include <stdio.h>
+
+#define EXPECT_ZERO(x) \
+    do { \
+        int __my_ret__ = x; \
+        if (__my_ret__) { \
+            int __my_errno__ = errno; \
+            fprintf(stderr, "TEST_ERROR: failed on line %d with return " \
+		    "code %d (errno: %d): got nonzero from %s\n", \
+		    __LINE__, __my_ret__, __my_errno__, #x); \
+            return __my_ret__; \
+        } \
+    } while (0);
+
+#define EXPECT_NULL(x) \
+    do { \
+        void* __my_ret__ = x; \
+        int __my_errno__ = errno; \
+        if (__my_ret__ != NULL) { \
+            fprintf(stderr, "TEST_ERROR: failed on line %d (errno: %d): " \
+		    "got non-NULL value %p from %s\n", \
+		    __LINE__, __my_errno__, __my_ret__, #x); \
+            return -1; \
+        } \
+    } while (0);
+
+#define EXPECT_NONNULL(x) \
+    do { \
+        void* __my_ret__ = x; \
+        int __my_errno__ = errno; \
+        if (__my_ret__ == NULL) { \
+            fprintf(stderr, "TEST_ERROR: failed on line %d (errno: %d): " \
+		    "got NULL from %s\n", __LINE__, __my_errno__, #x); \
+            return -1; \
+        } \
+    } while (0);
+
+#define EXPECT_NEGATIVE_ONE_WITH_ERRNO(x, e) \
+    do { \
+        int __my_ret__ = x; \
+        int __my_errno__ = errno; \
+        if (__my_ret__ != -1) { \
+            fprintf(stderr, "TEST_ERROR: failed on line %d with return " \
+                "code %d (errno: %d): expected -1 from %s\n", __LINE__, \
+                __my_ret__, __my_errno__, #x); \
+            return -1; \
+        } \
+        if (__my_errno__ != e) { \
+            fprintf(stderr, "TEST_ERROR: failed on line %d with return " \
+                "code %d (errno: %d): expected errno = %d from %s\n", \
+                __LINE__, __my_ret__, __my_errno__, e, #x); \
+            return -1; \
+	} \
+    } while (0);
+
+#define EXPECT_NONZERO(x) \
+    do { \
+        int __my_ret__ = x; \
+        int __my_errno__ = errno; \
+        if (__my_ret__) { \
+            fprintf(stderr, "TEST_ERROR: failed on line %d with return " \
+		    "code %d (errno: %d): got zero from %s\n", __LINE__, \
+                __my_ret__, __my_errno__, #x); \
+            return -1; \
+        } \
+    } while (0);
+
+#define EXPECT_NONNEGATIVE(x) \
+    do { \
+        int __my_ret__ = x; \
+        int __my_errno__ = errno; \
+        if (__my_ret__ < 0) { \
+            fprintf(stderr, "TEST_ERROR: failed on line %d with return " \
+                "code %d (errno: %d): got negative return from %s\n", \
+		    __LINE__, __my_ret__, __my_errno__, #x); \
+            return __my_ret__; \
+        } \
+    } while (0);
+
+#endif

Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.c?rev=1361440&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.c
(added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.c
Fri Jul 13 23:28:03 2012
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "hdfsJniHelper.h"
+#include "native_mini_dfs.h"
+
+#include <errno.h>
+#include <jni.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder"
+#define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster"
+#define HADOOP_CONF     "org/apache/hadoop/conf/Configuration"
+
+struct NativeMiniDfsCluster {
+    /**
+     * The NativeMiniDfsCluster object
+     */
+    jobject obj;
+};
+
+struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
+{
+    struct NativeMiniDfsCluster* cl = NULL;
+    jobject bld = NULL, bld2 = NULL, cobj = NULL;
+    jvalue  val;
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "nmdCreate: unable to construct JNIEnv.\n");
+        goto error;
+    }
+    cl = calloc(1, sizeof(struct NativeMiniDfsCluster));
+    if (!cl) {
+        fprintf(stderr, "nmdCreate: OOM");
+        goto error;
+    }
+    cobj = constructNewObjectOfClass(env, NULL, HADOOP_CONF, "()V");
+    if (!cobj) {
+        fprintf(stderr, "nmdCreate: unable to construct Configuration\n");
+        goto error_free_cl;
+    }
+    bld = constructNewObjectOfClass(env, NULL, MINIDFS_CLUSTER_BUILDER,
+                    "(L"HADOOP_CONF";)V", cobj);
+    if (!bld) {
+        fprintf(stderr, "nmdCreate: unable to construct "
+                "NativeMiniDfsCluster#Builder\n");
+        goto error_dlr_cobj;
+    }
+    if (invokeMethod(env, &val, NULL, INSTANCE, bld,
+            MINIDFS_CLUSTER_BUILDER, "format",
+            "(Z)L" MINIDFS_CLUSTER_BUILDER ";", conf->doFormat)) {
+        fprintf(stderr, "nmdCreate: failed to call Builder#doFormat\n");
+        goto error_dlr_bld;
+    }
+    bld2 = val.l;
+    if (invokeMethod(env, &val, NULL, INSTANCE, bld,
+            MINIDFS_CLUSTER_BUILDER, "build",
+            "()L" MINIDFS_CLUSTER ";")) {
+        fprintf(stderr, "nmdCreate: failed to call Builder#build\n");
+        goto error_dlr_bld2;
+    }
+	cl->obj = (*env)->NewGlobalRef(env, val.l);
+    if (!cl->obj) {
+        fprintf(stderr, "nmdCreate: failed to create global reference to "
+            "MiniDFSCluster\n");
+        goto error_dlr_val;
+    }
+    (*env)->DeleteLocalRef(env, val.l);
+    (*env)->DeleteLocalRef(env, bld2);
+    (*env)->DeleteLocalRef(env, bld);
+    (*env)->DeleteLocalRef(env, cobj);
+    return cl;
+
+error_dlr_val:
+    (*env)->DeleteLocalRef(env, val.l);
+error_dlr_bld2:
+    (*env)->DeleteLocalRef(env, bld2);
+error_dlr_bld:
+    (*env)->DeleteLocalRef(env, bld);
+error_dlr_cobj:
+    (*env)->DeleteLocalRef(env, cobj);
+error_free_cl:
+    free(cl);
+error:
+    return NULL;
+}
+
+void nmdFree(struct NativeMiniDfsCluster* cl)
+{
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "nmdFree: getJNIEnv failed\n");
+        free(cl);
+        return;
+    }
+    (*env)->DeleteGlobalRef(env, cl->obj);
+    free(cl);
+}
+
+int nmdShutdown(struct NativeMiniDfsCluster* cl)
+{
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "nmdShutdown: getJNIEnv failed\n");
+        return -EIO;
+    }
+    if (invokeMethod(env, NULL, NULL, INSTANCE, cl->obj,
+            MINIDFS_CLUSTER, "shutdown", "()V")) {
+        fprintf(stderr, "nmdShutdown: MiniDFSCluster#shutdown failure\n");
+        return -EIO;
+    }
+    return 0;
+}
+
+int nmdWaitClusterUp(struct NativeMiniDfsCluster *cl)
+{
+    JNIEnv *env = getJNIEnv();
+    if (!env) {
+        fprintf(stderr, "nmdWaitClusterUp: getJNIEnv failed\n");
+        return -EIO;
+    }
+    if (invokeMethod(env, NULL, NULL, INSTANCE, cl->obj,
+            MINIDFS_CLUSTER, "waitClusterUp", "()V")) {
+        fprintf(stderr, "nmdWaitClusterUp: MiniDFSCluster#waitClusterUp "
+                "failure\n");
+        return -EIO;
+    }
+    return 0;
+}
+
+int nmdGetNameNodePort(struct NativeMiniDfsCluster *cl)
+{
+    JNIEnv *env = getJNIEnv();
+    jvalue jVal;
+
+    if (!env) {
+        fprintf(stderr, "nmdHdfsConnect: getJNIEnv failed\n");
+        return -EIO;
+    }
+    // Note: this will have to be updated when HA nativeMiniDfs clusters are
+    // supported
+    if (invokeMethod(env, &jVal, NULL, INSTANCE, cl->obj,
+            MINIDFS_CLUSTER, "getNameNodePort", "()I")) {
+        fprintf(stderr, "nmdHdfsConnect: MiniDFSCluster#getNameNodePort "
+                "failure\n");
+        return -EIO;
+    }
+    return jVal.i;
+}

Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.h
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.h?rev=1361440&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.h
(added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/native_mini_dfs.h
Fri Jul 13 23:28:03 2012
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFS_NATIVE_MINI_DFS_H
+#define LIBHDFS_NATIVE_MINI_DFS_H
+
+#include <jni.h> /* for jboolean */
+
+struct NativeMiniDfsCluster; 
+
+/**
+ * Represents a configuration to use for creating a Native MiniDFSCluster
+ */
+struct NativeMiniDfsConf {
+    /**
+     * Nonzero if the cluster should be formatted prior to startup
+     */
+    jboolean doFormat;
+};
+
+/**
+ * Create a NativeMiniDfsBuilder
+ *
+ * @param conf      (inout) The cluster configuration
+ *
+ * @return      a NativeMiniDfsBuilder, or a NULL pointer on error.
+ */
+struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf);
+
+/**
+ * Wait until a MiniDFSCluster comes out of safe mode.
+ *
+ * @param cl        The cluster
+ *
+ * @return          0 on success; a non-zero error code if the cluster fails to
+ *                  come out of safe mode.
+ */
+int nmdWaitClusterUp(struct NativeMiniDfsCluster *cl);
+
+/**
+ * Shut down a NativeMiniDFS cluster
+ *
+ * @param cl        The cluster
+ *
+ * @return          0 on success; a non-zero error code if an exception is
+ *                  thrown.
+ */
+int nmdShutdown(struct NativeMiniDfsCluster *cl);
+
+/**
+ * Destroy a Native MiniDFSCluster
+ *
+ * @param cl        The cluster to destroy
+ */
+void nmdFree(struct NativeMiniDfsCluster* cl);
+
+/**
+ * Get the port that's in use by the given (non-HA) nativeMiniDfs
+ *
+ * @param cl        The initialized NativeMiniDfsCluster
+ *
+ * @return          the port, or a negative error code
+ */
+int nmdGetNameNodePort(struct NativeMiniDfsCluster *cl);
+
+#endif

Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_libhdfs_threaded.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_libhdfs_threaded.c?rev=1361440&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_libhdfs_threaded.c
(added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_libhdfs_threaded.c
Fri Jul 13 23:28:03 2012
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "expect.h"
+#include "hdfs.h"
+#include "native_mini_dfs.h"
+
+#include <errno.h>
+#include <semaphore.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define TLH_MAX_THREADS 100
+
+static sem_t tlhSem;
+
+static struct NativeMiniDfsCluster* tlhCluster;
+
+struct tlhThreadInfo {
+    /** Thread index */
+    int threadIdx;
+    /** 0 = thread was successful; error code otherwise */
+    int success;
+    /** pthread identifier */
+    pthread_t thread;
+};
+
+static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs)
+{
+    int ret, port;
+    hdfsFS hdfs;
+    
+    port = nmdGetNameNodePort(cl);
+    if (port < 0) {
+        fprintf(stderr, "hdfsSingleNameNodeConnect: nmdGetNameNodePort "
+                "returned error %d\n", port);
+        return port;
+    }
+    hdfs = hdfsConnectNewInstance("localhost", port);
+    if (!hdfs) {
+        ret = -errno;
+        return ret;
+    }
+    *fs = hdfs;
+    return 0;
+}
+
+static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs)
+{
+    char prefix[256], tmp[256];
+    hdfsFile file;
+    int ret, expected;
+
+    snprintf(prefix, sizeof(prefix), "/tlhData%04d", ti->threadIdx);
+
+    if (hdfsExists(fs, prefix) == 0) {
+        EXPECT_ZERO(hdfsDelete(fs, prefix, 1));
+    }
+    EXPECT_ZERO(hdfsCreateDirectory(fs, prefix));
+    snprintf(tmp, sizeof(tmp), "%s/file", prefix);
+
+    /* There should not be any file to open for reading. */
+    EXPECT_NULL(hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0));
+
+    file = hdfsOpenFile(fs, tmp, O_WRONLY, 0, 0, 0);
+    EXPECT_NONNULL(file);
+
+    /* TODO: implement writeFully and use it here */
+    expected = strlen(prefix);
+    ret = hdfsWrite(fs, file, prefix, expected);
+    if (ret < 0) {
+        ret = errno;
+        fprintf(stderr, "hdfsWrite failed and set errno %d\n", ret);
+        return ret;
+    }
+    if (ret != expected) {
+        fprintf(stderr, "hdfsWrite was supposed to write %d bytes, but "
+                "it wrote %d\n", ret, expected);
+        return EIO;
+    }
+    EXPECT_ZERO(hdfsFlush(fs, file));
+    EXPECT_ZERO(hdfsCloseFile(fs, file));
+
+    /* Let's re-open the file for reading */
+    file = hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0);
+    EXPECT_NONNULL(file);
+
+    /* TODO: implement readFully and use it here */
+    ret = hdfsRead(fs, file, tmp, sizeof(tmp));
+    if (ret < 0) {
+        ret = errno;
+        fprintf(stderr, "hdfsRead failed and set errno %d\n", ret);
+        return ret;
+    }
+    if (ret != expected) {
+        fprintf(stderr, "hdfsRead was supposed to read %d bytes, but "
+                "it read %d\n", ret, expected);
+        return EIO;
+    }
+    EXPECT_ZERO(memcmp(prefix, tmp, expected));
+    EXPECT_ZERO(hdfsCloseFile(fs, file));
+
+    // TODO: Non-recursive delete should fail?
+    //EXPECT_NONZERO(hdfsDelete(fs, prefix, 0));
+
+    EXPECT_ZERO(hdfsDelete(fs, prefix, 1));
+    return 0;
+}
+
+static void *testHdfsOperations(void *v)
+{
+    struct tlhThreadInfo *ti = (struct tlhThreadInfo*)v;
+    hdfsFS fs = NULL;
+    int ret;
+
+    fprintf(stderr, "testHdfsOperations(threadIdx=%d): starting\n",
+        ti->threadIdx);
+    ret = hdfsSingleNameNodeConnect(tlhCluster, &fs);
+    if (ret) {
+        fprintf(stderr, "testHdfsOperations(threadIdx=%d): "
+            "hdfsSingleNameNodeConnect failed with error %d.\n",
+            ti->threadIdx, ret);
+        ti->success = EIO;
+        return NULL;
+    }
+    ti->success = doTestHdfsOperations(ti, fs);
+    if (hdfsDisconnect(fs)) {
+        ret = errno;
+        fprintf(stderr, "hdfsDisconnect error %d\n", ret);
+        ti->success = ret;
+    }
+    return NULL;
+}
+
+static int checkFailures(struct tlhThreadInfo *ti, int tlhNumThreads)
+{
+    int i, threadsFailed = 0;
+    const char *sep = "";
+
+    for (i = 0; i < tlhNumThreads; i++) {
+        if (ti[i].success != 0) {
+            threadsFailed = 1;
+        }
+    }
+    if (!threadsFailed) {
+        fprintf(stderr, "testLibHdfs: all threads succeeded.  SUCCESS.\n");
+        return EXIT_SUCCESS;
+    }
+    fprintf(stderr, "testLibHdfs: some threads failed: [");
+    for (i = 0; i < tlhNumThreads; i++) {
+        if (ti[i].success != 0) {
+            fprintf(stderr, "%s%d", sep, i);
+            sep = ", "; 
+        }
+    }
+    fprintf(stderr, "].  FAILURE.\n");
+    return EXIT_FAILURE;
+}
+
+/**
+ * Test that we can write a file with libhdfs and then read it back
+ */
+int main(void)
+{
+    int i, tlhNumThreads;
+    const char *tlhNumThreadsStr;
+    struct tlhThreadInfo ti[TLH_MAX_THREADS];
+    struct NativeMiniDfsConf conf = {
+        .doFormat = 1,
+    };
+
+    tlhNumThreadsStr = getenv("TLH_NUM_THREADS");
+    if (!tlhNumThreadsStr) {
+        tlhNumThreadsStr = "3";
+    }
+    tlhNumThreads = atoi(tlhNumThreadsStr);
+    if ((tlhNumThreads <= 0) || (tlhNumThreads > TLH_MAX_THREADS)) {
+        fprintf(stderr, "testLibHdfs: must have a number of threads "
+                "between 1 and %d inclusive, not %d\n",
+                TLH_MAX_THREADS, tlhNumThreads);
+        return EXIT_FAILURE;
+    }
+    memset(&ti[0], 0, sizeof(ti));
+    for (i = 0; i < tlhNumThreads; i++) {
+        ti[i].threadIdx = i;
+    }
+
+    EXPECT_ZERO(sem_init(&tlhSem, 0, tlhNumThreads));
+    tlhCluster = nmdCreate(&conf);
+    EXPECT_NONNULL(tlhCluster);
+    EXPECT_ZERO(nmdWaitClusterUp(tlhCluster));
+
+    for (i = 0; i < tlhNumThreads; i++) {
+        EXPECT_ZERO(pthread_create(&ti[i].thread, NULL,
+            testHdfsOperations, &ti[i]));
+    }
+    for (i = 0; i < tlhNumThreads; i++) {
+        EXPECT_ZERO(pthread_join(ti[i].thread, NULL));
+    }
+
+    EXPECT_ZERO(nmdShutdown(tlhCluster));
+    nmdFree(tlhCluster);
+    EXPECT_ZERO(sem_destroy(&tlhSem));
+    return checkFailures(ti, tlhNumThreads);
+}

Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_native_mini_dfs.c
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_native_mini_dfs.c?rev=1361440&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_native_mini_dfs.c
(added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native/test_native_mini_dfs.c
Fri Jul 13 23:28:03 2012
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "expect.h"
+#include "native_mini_dfs.h"
+
+#include <errno.h>
+
+static struct NativeMiniDfsConf conf = {
+    .doFormat = 1,
+};
+
+/**
+ * Test that we can create a MiniDFSCluster and shut it down.
+ */
+int main(void) {
+    struct NativeMiniDfsCluster* cl;
+    
+    cl = nmdCreate(&conf);
+    EXPECT_NONNULL(cl);
+    EXPECT_ZERO(nmdWaitClusterUp(cl));
+    EXPECT_ZERO(nmdShutdown(cl));
+    nmdFree(cl);
+
+    return 0;
+}



Mime
View raw message