hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject git commit: YARN-2161. Fix build on macosx: YARN parts (Binglin Chang via aw)
Date Wed, 24 Sep 2014 15:49:01 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 151281bfc -> e5a531a8a


YARN-2161. Fix build on macosx: YARN parts (Binglin Chang via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5a531a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5a531a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5a531a8

Branch: refs/heads/branch-2
Commit: e5a531a8a7e7021ff709cc377629dbb8260b3503
Parents: 151281b
Author: Allen Wittenauer <aw@apache.org>
Authored: Wed Sep 24 08:48:42 2014 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Wed Sep 24 08:48:42 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  2 +
 .../src/CMakeLists.txt                          |  3 +
 .../src/config.h.cmake                          |  2 +
 .../container-executor/impl/configuration.c     |  7 +-
 .../impl/container-executor.c                   | 83 ++++++++++++++------
 .../test/test-container-executor.c              | 16 ++--
 6 files changed, 80 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5a531a8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 15df498..d3fe962 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -404,6 +404,8 @@ Release 2.6.0 - UNRELEASED
     TestFairScheduler.testContinuousScheduling. 
     (Ratandeep Ratti and kasha via kasha)
 
+    YARN-2161. Fix build on macosx: YARN parts (Binglin Chang via aw)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5a531a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index d69b845..f691a9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -20,6 +20,9 @@ set(CMAKE_BUILD_TYPE, Release)
 
 include(../../../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
 
+include(CheckFunctionExists)
+CHECK_FUNCTION_EXISTS(fcloseall HAVE_FCLOSEALL)
+
 function(output_directory TGT DIR)
     SET_TARGET_PROPERTIES(${TGT} PROPERTIES
         RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5a531a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake
index 4b59531..0f7a490 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/config.h.cmake
@@ -20,4 +20,6 @@
 
 #cmakedefine HADOOP_CONF_DIR "@HADOOP_CONF_DIR@"
 
+#cmakedefine HAVE_FCLOSEALL "@HAVE_FCLOSEALL@"
+
 #endif

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5a531a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index f393c97..15b53ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -103,7 +103,12 @@ char *resolve_config_path(const char* file_name, const char *root) {
     real_fname = buffer;
   }
 
-  return (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
+  char * ret = (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
+#ifdef DEBUG
+  fprintf(stderr, "resolve_config_path(file_name=%s,root=%s)=%s\n",
+          file_name, root ? root : "null", ret ? ret : "null");
+#endif
+  return ret;
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5a531a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index b64da19..f582d85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -19,6 +19,7 @@
 #include "configuration.h"
 #include "container-executor.h"
 
+#include <libgen.h>
 #include <dirent.h>
 #include <fcntl.h>
 #include <fts.h>
@@ -450,39 +451,59 @@ char *get_tmp_directory(const char *work_dir) {
  * with the desired permissions.
  */
 int mkdirs(const char* path, mode_t perm) {
-  char *buffer = strdup(path);
-  char *token;
-  int cwd = open("/", O_RDONLY);
-  if (cwd == -1) {
-    fprintf(LOGFILE, "Can't open / in %s - %s\n", path, strerror(errno));
-    free(buffer);
+  struct stat sb;
+  char * npath;
+  char * p;
+  if (stat(path, &sb) == 0) {
+    if (S_ISDIR (sb.st_mode)) {
+      return 0;
+    } else {
+      fprintf(LOGFILE, "Path %s is file not dir\n", path);
+      return -1;
+    }
+  }
+  npath = strdup(path);
+  if (npath == NULL) {
+    fprintf(LOGFILE, "Not enough memory to copy path string");
     return -1;
   }
-  for(token = strtok(buffer, "/"); token != NULL; token = strtok(NULL, "/")) {
-    if (mkdirat(cwd, token, perm) != 0) {
-      if (errno != EEXIST) {
-        fprintf(LOGFILE, "Can't create directory %s in %s - %s\n", 
-                token, path, strerror(errno));
-        close(cwd);
-        free(buffer);
+  /* Skip leading slashes. */
+  p = npath;
+  while (*p == '/') {
+    p++;
+  }
+
+  while (NULL != (p = strchr(p, '/'))) {
+    *p = '\0';
+    if (stat(npath, &sb) != 0) {
+      if (mkdir(npath, perm) != 0) {
+        fprintf(LOGFILE, "Can't create directory %s in %s - %s\n", npath,
+                path, strerror(errno));
+        free(npath);
         return -1;
       }
-    }
-    int new_dir = openat(cwd, token, O_RDONLY);
-    close(cwd);
-    cwd = new_dir;
-    if (cwd == -1) {
-      fprintf(LOGFILE, "Can't open %s in %s - %s\n", token, path, 
-              strerror(errno));
-      free(buffer);
+    } else if (!S_ISDIR (sb.st_mode)) {
+      fprintf(LOGFILE, "Path %s is file not dir\n", npath);
+      free(npath);
       return -1;
     }
+    *p++ = '/'; /* restore slash */
+    while (*p == '/')
+      p++;
+  }
+
+  /* Create the final directory component. */
+  if (mkdir(npath, perm) != 0) {
+    fprintf(LOGFILE, "Can't create directory %s - %s\n", npath,
+            strerror(errno));
+    free(npath);
+    return -1;
   }
-  free(buffer);
-  close(cwd);
+  free(npath);
   return 0;
 }
 
+
 /**
  * Function to prepare the container directories.
  * It creates the container work and log directories.
@@ -582,7 +603,7 @@ int is_whitelisted(const char *user) {
   char **users = whitelist;
   if (whitelist != NULL) {
     for(; *users; ++users) {
-      if (strncmp(*users, user, LOGIN_NAME_MAX) == 0) {
+      if (strncmp(*users, user, sysconf(_SC_LOGIN_NAME_MAX)) == 0) {
         free_values(whitelist);
         return 1;
       }
@@ -1068,7 +1089,16 @@ int launch_container_as_user(const char *user, const char *app_id,
     goto cleanup;
   }
 
+#if HAVE_FCLOSEALL
   fcloseall();
+#else
+  // only those fds are opened assuming no bug
+  fclose(LOGFILE);
+  fclose(ERRORFILE);
+  fclose(stdin);
+  fclose(stdout);
+  fclose(stderr);
+#endif
   umask(0027);
   if (chdir(work_dir) != 0) {
     fprintf(LOGFILE, "Can't change directory to %s -%s\n", work_dir,
@@ -1351,6 +1381,10 @@ void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid)
{
  * hierarchy: the top directory of the hierarchy for the NM
  */
 int mount_cgroup(const char *pair, const char *hierarchy) {
+#ifndef __linux
+  fprintf(LOGFILE, "Failed to mount cgroup controller, not supported\n");
+  return -1;
+#else
   char *controller = malloc(strlen(pair));
   char *mount_path = malloc(strlen(pair));
   char hier_path[PATH_MAX];
@@ -1387,5 +1421,6 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
   free(mount_path);
 
   return result;
+#endif
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5a531a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index e9a47b1..e9ac234 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -98,7 +98,7 @@ int write_config_file(char *file_name) {
   }
   fprintf(file, "banned.users=bannedUser\n");
   fprintf(file, "min.user.id=500\n");
-  fprintf(file, "allowed.system.users=allowedUser,bin\n");
+  fprintf(file, "allowed.system.users=allowedUser,daemon\n");
   fclose(file);
   return 0;
 }
@@ -222,20 +222,20 @@ void test_check_user() {
     printf("FAIL: failed check for system user root\n");
     exit(1);
   }
-  if (check_user("bin") == NULL) {
-    printf("FAIL: failed check for whitelisted system user bin\n");
+  if (check_user("daemon") == NULL) {
+    printf("FAIL: failed check for whitelisted system user daemon\n");
     exit(1);
   }
 }
 
 void test_resolve_config_path() {
   printf("\nTesting resolve_config_path\n");
-  if (strcmp(resolve_config_path("/etc/passwd", NULL), "/etc/passwd") != 0) {
-    printf("FAIL: failed to resolve config_name on an absolute path name: /etc/passwd\n");
+  if (strcmp(resolve_config_path("/bin/ls", NULL), "/bin/ls") != 0) {
+    printf("FAIL: failed to resolve config_name on an absolute path name: /bin/ls\n");
     exit(1);
   }
-  if (strcmp(resolve_config_path("../etc/passwd", "/etc/passwd"), "/etc/passwd") != 0) {
-    printf("FAIL: failed to resolve config_name on a relative path name: ../etc/passwd (relative
to /etc/passwd)");
+  if (strcmp(resolve_config_path("../bin/ls", "/bin/ls"), "/bin/ls") != 0) {
+    printf("FAIL: failed to resolve config_name on a relative path name: ../bin/ls (relative
to /bin/ls)");
     exit(1);
   }
 }
@@ -476,7 +476,7 @@ void test_signal_container_group() {
     printf("FAIL: fork failed\n");
     exit(1);
   } else if (child == 0) {
-    setpgrp();
+    setpgid(0,0);
     if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) {
       exit(1);
     }


Mime
View raw message