hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject [23/50] [abbrv] hadoop git commit: HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs. Contributed by Zhanwei Wang.
Date Tue, 13 Jan 2015 17:50:24 GMT
HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs. Contributed by Zhanwei
Wang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/708b1aa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/708b1aa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/708b1aa0

Branch: refs/heads/HDFS-6994
Commit: 708b1aa074d98df7d8abdc805818c787ce5779b8
Parents: 20625c8
Author: cnauroth <cnauroth@apache.org>
Authored: Thu Jan 8 10:23:40 2015 -0800
Committer: cnauroth <cnauroth@apache.org>
Committed: Thu Jan 8 10:23:40 2015 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 ++
 .../hadoop-hdfs/src/CMakeLists.txt              |  8 ++-
 .../src/main/native/libhdfs/native_mini_dfs.c   | 52 ++++++++++----------
 .../src/main/native/libhdfs/native_mini_dfs.h   | 17 +++++--
 .../native/libhdfs/test/test_libhdfs_zerocopy.c | 31 ++++++++++++
 5 files changed, 80 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9f48c76..9c1be4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED
     HDFS-7561. TestFetchImage should write fetched-image-dir under target.
     (Liang Xie via shv)
 
+    HDFS-7589. Break the dependency between libnative_mini_dfs and libhdfs.
+    (Zhanwei Wang via cnauroth)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
index 227be45..8da49c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
@@ -172,9 +172,15 @@ target_link_libraries(test_libhdfs_write
 
 add_library(native_mini_dfs
     main/native/libhdfs/native_mini_dfs.c
+    main/native/libhdfs/common/htable.c
+    main/native/libhdfs/exception.c
+    main/native/libhdfs/jni_helper.c
+    ${OS_DIR}/mutexes.c
+    ${OS_DIR}/thread_local_storage.c
 )
 target_link_libraries(native_mini_dfs
-    hdfs
+    ${JAVA_JVM_LIBRARY}
+    ${OS_LINK_LIBRARIES}
 )
 
 add_executable(test_native_mini_dfs

http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
index 2c42fa5..b37ebcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.c
@@ -17,8 +17,6 @@
  */
 
 #include "exception.h"
-#include "hdfs.h"
-#include "hdfs_test.h"
 #include "jni_helper.h"
 #include "native_mini_dfs.h"
 #include "platform.h"
@@ -32,6 +30,10 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+#ifndef EINTERNAL
+#define EINTERNAL 255
+#endif
+
 #define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder"
 #define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster"
 #define HADOOP_CONF     "org/apache/hadoop/conf/Configuration"
@@ -52,6 +54,25 @@ struct NativeMiniDfsCluster {
     char domainSocketPath[PATH_MAX];
 };
 
+static int hdfsDisableDomainSocketSecurity(void)
+{
+    jthrowable jthr;
+    JNIEnv* env = getJNIEnv();
+    if (env == NULL) {
+      errno = EINTERNAL;
+      return -1;
+    }
+    jthr = invokeMethod(env, NULL, STATIC, NULL,
+            "org/apache/hadoop/net/unix/DomainSocket",
+            "disableBindPathValidation", "()V");
+    if (jthr) {
+        errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+            "DomainSocket#disableBindPathValidation");
+        return -1;
+    }
+    return 0;
+}
+
 static jthrowable nmdConfigureShortCircuit(JNIEnv *env,
               struct NativeMiniDfsCluster *cl, jobject cobj)
 {
@@ -345,29 +366,10 @@ error_dlr_nn:
     return ret;
 }
 
-int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
-                            struct hdfsBuilder *bld)
-{
-    int ret;
-    tPort port;
-
-    hdfsBuilderSetNameNode(bld, "localhost");
-    port = (tPort)nmdGetNameNodePort(cl);
-    if (port < 0) {
-      fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port);
-      return EIO;
-    }
-    hdfsBuilderSetNameNodePort(bld, port);
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl) {
     if (cl->domainSocketPath[0]) {
-      ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true");
-      if (ret) {
-          return ret;
-      }
-      ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path",
-                            cl->domainSocketPath);
-      if (ret) {
-          return ret;
-      }
+        return cl->domainSocketPath;
     }
-    return 0;
+
+    return NULL;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
index 41d69c2..ce8b1cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/native_mini_dfs.h
@@ -21,6 +21,10 @@
 
 #include <jni.h> /* for jboolean */
 
+#ifdef __cplusplus
+extern  "C" {
+#endif
+
 struct hdfsBuilder;
 struct NativeMiniDfsCluster; 
 
@@ -110,13 +114,16 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
                                int *port, const char **hostName);
 
 /**
- * Configure the HDFS builder appropriately to connect to this cluster.
+ * Get domain socket path set for this cluster.
  *
- * @param bld       The hdfs builder
+ * @param cl        The cluster
  *
- * @return          the port, or a negative error code
+ * @return          A const string of domain socket path, or NULL if not set.
  */
-int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
-                            struct hdfsBuilder *bld);
+const char *hdfsGetDomainSocketPath(const struct NativeMiniDfsCluster *cl);
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif

http://git-wip-us.apache.org/repos/asf/hadoop/blob/708b1aa0/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
index 3774417..92941cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test/test_libhdfs_zerocopy.c
@@ -201,6 +201,37 @@ static int createZeroCopyTestFile(hdfsFS fs, char *testFileName,
     return 0;
 }
 
+static int nmdConfigureHdfsBuilder(struct NativeMiniDfsCluster *cl,
+                            struct hdfsBuilder *bld) {
+    int ret;
+    tPort port;
+    const char *domainSocket;
+
+    hdfsBuilderSetNameNode(bld, "localhost");
+    port = (tPort) nmdGetNameNodePort(cl);
+    if (port < 0) {
+      fprintf(stderr, "nmdGetNameNodePort failed with error %d\n", -port);
+      return EIO;
+    }
+    hdfsBuilderSetNameNodePort(bld, port);
+
+    domainSocket = hdfsGetDomainSocketPath(cl);
+
+    if (domainSocket) {
+      ret = hdfsBuilderConfSetStr(bld, "dfs.client.read.shortcircuit", "true");
+      if (ret) {
+        return ret;
+      }
+      ret = hdfsBuilderConfSetStr(bld, "dfs.domain.socket.path",
+                                  domainSocket);
+      if (ret) {
+        return ret;
+      }
+    }
+    return 0;
+}
+
+
 /**
  * Test that we can write a file with libhdfs and then read it back
  */


Mime
View raw message