From common-commits-return-77552-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Thu Jan 11 19:28:49 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id D7B96180656 for ; Thu, 11 Jan 2018 19:28:49 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id C68F1160C23; Thu, 11 Jan 2018 18:28:49 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E5AA3160C13 for ; Thu, 11 Jan 2018 19:28:48 +0100 (CET) Received: (qmail 63741 invoked by uid 500); 11 Jan 2018 18:28:47 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 63732 invoked by uid 99); 11 Jan 2018 18:28:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 Jan 2018 18:28:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8715CED31C; Thu, 11 Jan 2018 18:28:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: szegedim@apache.org To: common-commits@hadoop.apache.org Message-Id: <24b11b3436624b1bbf9ae2d24977a0f3@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-7590. Improve container-executor validation check. Contributed by Eric Yang. Date: Thu, 11 Jan 2018 18:28:36 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-3.0 a13a773ed -> ed9c516bc YARN-7590. Improve container-executor validation check. Contributed by Eric Yang. (cherry picked from commit bc285da) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed9c516b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed9c516b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed9c516b Branch: refs/heads/branch-3.0 Commit: ed9c516bc87accb90e58a3b919ef21779f72be50 Parents: a13a773 Author: Miklos Szegedi Authored: Thu Jan 11 09:40:09 2018 -0800 Committer: Miklos Szegedi Committed: Thu Jan 11 10:28:08 2018 -0800 ---------------------------------------------------------------------- .../impl/container-executor.c | 41 +++++++++++++++++- .../impl/container-executor.h | 5 +++ .../test/test-container-executor.c | 45 ++++++++++++++++---- 3 files changed, 82 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed9c516b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 3b04f88..0447b8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -517,10 +517,32 @@ char *get_app_directory(const char * nm_root, const char *user, * Get the user directory of a particular user */ char *get_user_directory(const char *nm_root, const char *user) { + int result = check_nm_local_dir(nm_uid, nm_root); + if (result != 0) { + return NULL; + } return concatenate(USER_DIR_PATTERN, "user_dir_path", 2, nm_root, user); } /** + * Check node manager local dir permission. + */ +int check_nm_local_dir(uid_t caller_uid, const char *nm_root) { + struct stat info; + errno = 0; + int err = stat(nm_root, &info); + if (err < 0) { + fprintf(LOGFILE, "Error checking file stats for %s %d %s.\n", nm_root, err, strerror(errno)); + return 1; + } + if (caller_uid != info.st_uid) { + fprintf(LOGFILE, "Permission mismatch for %s for caller uid: %d, owner uid: %d.\n", nm_root, caller_uid, info.st_uid); + return 1; + } + return 0; +} + +/** * Get the container directory for the given container_id */ char *get_container_work_directory(const char *nm_root, const char *user, @@ -666,6 +688,11 @@ static int create_container_directories(const char* user, const char *app_id, for(local_dir_ptr = local_dir; *local_dir_ptr != NULL; ++local_dir_ptr) { char *container_dir = get_container_work_directory(*local_dir_ptr, user, app_id, container_id); + int check = check_nm_local_dir(nm_uid, *local_dir_ptr); + if (check != 0) { + free(container_dir); + continue; + } if (container_dir == NULL) { return OUT_OF_MEMORY; } @@ -692,6 +719,14 @@ static int create_container_directories(const char* user, const char *app_id, char* const* log_dir_ptr; for(log_dir_ptr = log_dir; *log_dir_ptr != NULL; ++log_dir_ptr) { char *container_log_dir = get_app_log_directory(*log_dir_ptr, combined_name); + int check = check_nm_local_dir(nm_uid, *log_dir_ptr); + if (check != 0) { + container_log_dir = NULL; + } + if (strstr(container_log_dir, "..") != 0) { + fprintf(LOGFILE, "Unsupported container log directory path detected.\n"); + container_log_dir = NULL; + } if (container_log_dir == NULL) { free(combined_name); return OUT_OF_MEMORY; @@ -1032,6 +1067,10 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) { char *any_one_app_log_dir = NULL; for(log_root=log_dirs; *log_root != NULL; ++log_root) { char *app_log_dir = get_app_log_directory(*log_root, app_id); + int result = check_nm_local_dir(nm_uid, *log_root); + if (result != 0) { + app_log_dir = NULL; + } if (app_log_dir == NULL) { // try the next one } else if (create_directory_for_user(app_log_dir) != 0) { @@ -2213,4 +2252,4 @@ int traffic_control_read_stats(char *command_file) { */ struct configuration* get_cfg() { return &CFG; -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed9c516b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 6d4220f..c5f3d99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -178,6 +178,11 @@ char *get_user_directory(const char *nm_root, const char *user); char *get_app_directory(const char * nm_root, const char *user, const char *app_id); +/** + * Check node manager local dir permission. + */ +int check_nm_local_dir(uid_t caller_uid, const char *nm_root); + char *get_container_work_directory(const char *nm_root, const char *user, const char *app_id, const char *container_id); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed9c516b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 4e8db6c..1a6d4e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -37,6 +37,7 @@ static char* username = NULL; static char* yarn_username = NULL; static char** local_dirs = NULL; static char** log_dirs = NULL; +static uid_t nm_uid = -1; /** * Run the command using the effective user id. @@ -152,8 +153,8 @@ void check_pid_file(const char* pid_file, pid_t mypid) { } void test_get_user_directory() { - char *user_dir = get_user_directory(TMPDIR, "user"); - char *expected = TMPDIR "/usercache/user"; + char *user_dir = get_user_directory(TEST_ROOT, "user"); + char *expected = TEST_ROOT "/usercache/user"; if (strcmp(user_dir, expected) != 0) { printf("test_get_user_directory expected %s got %s\n", expected, user_dir); exit(1); @@ -161,9 +162,32 @@ void test_get_user_directory() { free(user_dir); } +void test_check_nm_local_dir() { + // check filesystem is same as running user. + int expected = 0; + char *local_path = TEST_ROOT "target"; + char *root_path = "/"; + if (mkdirs(local_path, 0700) != 0) { + printf("FAIL: unble to create node manager local directory: %s\n", local_path); + exit(1); + } + int actual = check_nm_local_dir(nm_uid, local_path); + if (expected != actual) { + printf("test_nm_local_dir expected %d got %d\n", expected, actual); + exit(1); + } + // check filesystem is different from running user. + expected = 1; + actual = check_nm_local_dir(nm_uid, root_path); + if (expected != actual && nm_uid != 0) { + printf("test_nm_local_dir expected %d got %d\n", expected, actual); + exit(1); + } +} + void test_get_app_directory() { - char *expected = TMPDIR "/usercache/user/appcache/app_200906101234_0001"; - char *app_dir = (char *) get_app_directory(TMPDIR, "user", + char *expected = TEST_ROOT "/usercache/user/appcache/app_200906101234_0001"; + char *app_dir = (char *) get_app_directory(TEST_ROOT, "user", "app_200906101234_0001"); if (strcmp(app_dir, expected) != 0) { printf("test_get_app_directory expected %s got %s\n", expected, app_dir); @@ -173,9 +197,9 @@ void test_get_app_directory() { } void test_get_container_directory() { - char *container_dir = get_container_work_directory(TMPDIR, "owen", "app_1", + char *container_dir = get_container_work_directory(TEST_ROOT, "owen", "app_1", "container_1"); - char *expected = TMPDIR"/usercache/owen/appcache/app_1/container_1"; + char *expected = TEST_ROOT "/usercache/owen/appcache/app_1/container_1"; if (strcmp(container_dir, expected) != 0) { printf("Fail get_container_work_directory got %s expected %s\n", container_dir, expected); @@ -185,9 +209,9 @@ void test_get_container_directory() { } void test_get_container_launcher_file() { - char *expected_file = (TMPDIR"/usercache/user/appcache/app_200906101234_0001" + char *expected_file = (TEST_ROOT "/usercache/user/appcache/app_200906101234_0001" "/launch_container.sh"); - char *app_dir = get_app_directory(TMPDIR, "user", + char *app_dir = get_app_directory(TEST_ROOT, "user", "app_200906101234_0001"); char *container_file = get_container_launcher_file(app_dir); if (strcmp(container_file, expected_file) != 0) { @@ -1182,6 +1206,8 @@ int main(int argc, char **argv) { LOGFILE = stdout; ERRORFILE = stderr; + nm_uid = getuid(); + printf("Attempting to clean up from any previous runs\n"); // clean up any junk from previous run if (system("chmod -R u=rwx " TEST_ROOT "; rm -fr " TEST_ROOT)) { @@ -1232,6 +1258,9 @@ int main(int argc, char **argv) { printf("\nTesting get_user_directory()\n"); test_get_user_directory(); + printf("\nTesting check_nm_local_dir()\n"); + test_check_nm_local_dir(); + printf("\nTesting get_app_directory()\n"); test_get_app_directory(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org