From mapreduce-commits-return-1177-apmail-hadoop-mapreduce-commits-archive=hadoop.apache.org@hadoop.apache.org Thu Jan 06 19:59:35 2011 Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 98490 invoked from network); 6 Jan 2011 19:59:35 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 6 Jan 2011 19:59:35 -0000 Received: (qmail 53509 invoked by uid 500); 6 Jan 2011 19:59:35 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 53472 invoked by uid 500); 6 Jan 2011 19:59:35 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 53464 invoked by uid 99); 6 Jan 2011 19:59:35 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Jan 2011 19:59:35 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Jan 2011 19:59:33 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id D506323888FD; Thu, 6 Jan 2011 19:59:13 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1056046 - in /hadoop/mapreduce/trunk: ./ src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/server/tasktracker/ src/test/mapred/org/apache/hadoop/mapred/ src/test/unit/org/apache/hadoop/mapred/ Date: Thu, 06 Jan 2011 19:59:13 -0000 To: mapreduce-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110106195913.D506323888FD@eris.apache.org> Author: todd Date: Thu Jan 6 19:59:13 2011 New Revision: 1056046 URL: http://svn.apache.org/viewvc?rev=1056046&view=rev Log: MAPREDUCE-2234. If Localizer can't create task log directory, it should fail on the spot. Contributed by Todd Lipcon Added: hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java Modified: hadoop/mapreduce/trunk/CHANGES.txt hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java Modified: hadoop/mapreduce/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1056046&r1=1056045&r2=1056046&view=diff ============================================================================== --- hadoop/mapreduce/trunk/CHANGES.txt (original) +++ hadoop/mapreduce/trunk/CHANGES.txt Thu Jan 6 19:59:13 2011 @@ -460,6 +460,9 @@ Release 0.22.0 - Unreleased MAPREDUCE-2096. Secure local filesystem IO from symlink vulnerabilities (todd) + MAPREDUCE-2234. If Localizer can't create task log directory, it should fail + on the spot. (todd) + Release 0.21.1 - Unreleased NEW FEATURES Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java?rev=1056046&r1=1056045&r2=1056046&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskController.java Thu Jan 6 19:59:13 2011 @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.CleanupQ import org.apache.hadoop.mapred.JvmManager.JvmEnv; import org.apache.hadoop.mapreduce.server.tasktracker.Localizer; import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.classification.InterfaceAudience; @@ -80,7 +81,7 @@ public abstract class TaskController imp for (String localDir : this.mapredLocalDirs) { // Set up the mapreduce.cluster.local.directories. File mapredlocalDir = new File(localDir); - if (!mapredlocalDir.exists() && !mapredlocalDir.mkdirs()) { + if (!mapredlocalDir.isDirectory() && !mapredlocalDir.mkdirs()) { LOG.warn("Unable to create mapreduce.cluster.local.directory : " + mapredlocalDir.getPath()); } else { @@ -91,12 +92,13 @@ public abstract class TaskController imp // Set up the user log directory File taskLog = TaskLog.getUserLogDir(); - if (!taskLog.exists() && !taskLog.mkdirs()) { + if (!taskLog.isDirectory() && !taskLog.mkdirs()) { LOG.warn("Unable to create taskLog directory : " + taskLog.getPath()); } else { Localizer.PermissionsHandler.setPermissions(taskLog, Localizer.PermissionsHandler.sevenFiveFive); } + DiskChecker.checkDir(TaskLog.getUserLogDir()); } /** Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java?rev=1056046&r1=1056045&r2=1056046&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java (original) +++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java Thu Jan 6 19:59:13 2011 @@ -365,13 +365,13 @@ public class Localizer { * * @param jobId */ - public void initializeJobLogDir(JobID jobId) { + public void initializeJobLogDir(JobID jobId) throws IOException { File jobUserLogDir = TaskLog.getJobDir(jobId); if (!jobUserLogDir.exists()) { boolean ret = jobUserLogDir.mkdirs(); if (!ret) { - LOG.warn("Could not create job user log directory: " + jobUserLogDir); - return; + throw new IOException("Could not create job user log directory: " + + jobUserLogDir); } } Localizer.PermissionsHandler.setPermissions(jobUserLogDir, Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java?rev=1056046&r1=1056045&r2=1056046&view=diff ============================================================================== --- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java (original) +++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerLocalization.java Thu Jan 6 19:59:13 2011 @@ -468,6 +468,36 @@ public class TestTaskTrackerLocalization checkJobLocalization(); } + /** + * Test that, if the job log dir can't be created, the job will fail + * during localization rather than at the time when the task itself + * tries to write into it. + */ + public void testJobLocalizationFailsIfLogDirUnwritable() + throws Exception { + if (!canRun()) { + return; + } + + File logDir = TaskLog.getJobDir(jobId); + File logDirParent = logDir.getParentFile(); + + try { + assertTrue(logDirParent.mkdirs() || logDirParent.isDirectory()); + FileUtil.fullyDelete(logDir); + FileUtil.chmod(logDirParent.getAbsolutePath(), "000"); + + tracker.localizeJob(tip); + fail("No exception"); + } catch (IOException ioe) { + LOG.info("Got exception", ioe); + assertTrue(ioe.getMessage().contains("Could not create job user log")); + } finally { + // Put it back just to be safe + FileUtil.chmod(logDirParent.getAbsolutePath(), "755"); + } + } + protected void checkJobLocalization() throws IOException { // Check the directory structure Added: hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java?rev=1056046&view=auto ============================================================================== --- hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java (added) +++ hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestTaskTrackerDirectories.java Thu Jan 6 19:59:13 2011 @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.mapreduce.MRConfig; +import org.junit.Test; +import org.junit.Before; + +/** + * Tests for the correct behavior of the TaskTracker starting up with + * respect to its local-disk directories. + */ +public class TestTaskTrackerDirectories { + private final String TEST_DIR = new File("build/test/testmapredlocaldir") + .getAbsolutePath(); + + @Before + public void deleteTestDir() throws IOException { + FileUtil.fullyDelete(new File(TEST_DIR)); + assertFalse("Could not delete " + TEST_DIR, + new File(TEST_DIR).exists()); + } + + @Test + public void testCreatesLocalDirs() throws Exception { + Configuration conf = new Configuration(); + String[] dirs = new String[] { + TEST_DIR + "/local1", + TEST_DIR + "/local2" + }; + + conf.setStrings(MRConfig.LOCAL_DIR, dirs); + setupTaskController(conf); + + for (String dir : dirs) { + checkDir(dir); + } + } + + @Test + public void testFixesLocalDirPermissions() throws Exception { + Configuration conf = new Configuration(); + String[] dirs = new String[] { + TEST_DIR + "/badperms" + }; + + new File(dirs[0]).mkdirs(); + FileUtil.chmod(dirs[0], "000"); + + conf.setStrings(MRConfig.LOCAL_DIR, dirs); + setupTaskController(conf); + + for (String dir : dirs) { + checkDir(dir); + } + } + + @Test + public void testCreatesLogDir() throws Exception { + File dir = TaskLog.getUserLogDir(); + FileUtil.fullyDelete(dir); + + setupTaskController(new Configuration()); + + checkDir(dir.getAbsolutePath()); + } + + /** + * If the log dir can't be created, the TT should fail to start since + * it will be unable to localize or run tasks. + */ + @Test + public void testCantCreateLogDir() throws Exception { + File dir = TaskLog.getUserLogDir(); + FileUtil.fullyDelete(dir); + assertTrue("Making file in place of log dir", + dir.createNewFile()); + + try { + setupTaskController(new Configuration()); + fail("Didn't throw!"); + } catch (IOException ioe) { + System.err.println("Got expected exception"); + ioe.printStackTrace(System.out); + } + } + + @Test + public void testFixesLogDirPermissions() throws Exception { + File dir = TaskLog.getUserLogDir(); + FileUtil.fullyDelete(dir); + dir.mkdirs(); + FileUtil.chmod(dir.getAbsolutePath(), "000"); + + setupTaskController(new Configuration()); + + checkDir(dir.getAbsolutePath()); + } + + private void setupTaskController(Configuration conf) throws IOException { + TaskController tc = new DefaultTaskController(); + tc.setConf(conf); + tc.setup(); + } + + private void checkDir(String dir) throws IOException { + FileSystem fs = RawLocalFileSystem.get(new Configuration()); + File f = new File(dir); + assertTrue(dir + "should exist", f.exists()); + FileStatus stat = fs.getFileStatus(new Path(dir)); + assertEquals(dir + " has correct permissions", + 0755, stat.getPermission().toShort()); + } +}