Return-Path: X-Original-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-yarn-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4CF0D11DA0 for ; Sat, 24 May 2014 01:24:52 +0000 (UTC) Received: (qmail 23433 invoked by uid 500); 24 May 2014 01:24:52 -0000 Delivered-To: apmail-hadoop-yarn-commits-archive@hadoop.apache.org Received: (qmail 23390 invoked by uid 500); 24 May 2014 01:24:52 -0000 Mailing-List: contact yarn-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: yarn-commits@hadoop.apache.org Delivered-To: mailing list yarn-commits@hadoop.apache.org Received: (qmail 23382 invoked by uid 99); 24 May 2014 01:24:52 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 24 May 2014 01:24:52 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 24 May 2014 01:24:51 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id BC857238890D; Sat, 24 May 2014 01:24:26 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1597231 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/ hadoop-yarn/hadoop-yarn-s... Date: Sat, 24 May 2014 01:24:26 -0000 To: yarn-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140524012426.BC857238890D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Sat May 24 01:24:26 2014 New Revision: 1597231 URL: http://svn.apache.org/r1597231 Log: YARN-2071. Modified levelDB store permissions to be readable only by the server user. Contributed by Zhijie Shen. Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1597231&r1=1597230&r2=1597231&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Sat May 24 01:24:26 2014 @@ -108,6 +108,9 @@ Release 2.5.0 - UNRELEASED YARN-2073. Fair Scheduler: Add a utilization threshold to prevent preempting resources when cluster is free (Karthik Kambatla via Sandy Ryza) + YARN-2071. Modified levelDB store permissions to be readable only by the + server user. (Zhijie Shen via vinodkv) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java?rev=1597231&r1=1597230&r2=1597231&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/LeveldbTimelineStore.java Sat May 24 01:24:26 2014 @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -36,13 +39,16 @@ import java.util.TreeMap; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.collections.map.LRUMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.service.AbstractService; @@ -50,8 +56,8 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.fusesource.leveldbjni.JniDBFactory; @@ -62,8 +68,7 @@ import org.iq80.leveldb.ReadOptions; import org.iq80.leveldb.WriteBatch; import org.iq80.leveldb.WriteOptions; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.readReverseOrderedLong; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import com.google.common.annotations.VisibleForTesting; /** *

An implementation of an application timeline store backed by leveldb.

@@ -120,7 +125,9 @@ public class LeveldbTimelineStore extend private static final Log LOG = LogFactory .getLog(LeveldbTimelineStore.class); - private static final String FILENAME = "leveldb-timeline-store.ldb"; + @Private + @VisibleForTesting + static final String FILENAME = "leveldb-timeline-store.ldb"; private static final byte[] START_TIME_LOOKUP_PREFIX = "k".getBytes(); private static final byte[] ENTITY_ENTRY_PREFIX = "e".getBytes(); @@ -135,6 +142,11 @@ public class LeveldbTimelineStore extend private static final byte[] EMPTY_BYTES = new byte[0]; + @Private + @VisibleForTesting + static final FsPermission LEVELDB_DIR_UMASK = FsPermission + .createImmutable((short) 0700); + private Map startTimeWriteCache; private Map startTimeReadCache; @@ -164,16 +176,23 @@ public class LeveldbTimelineStore extend YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); JniDBFactory factory = new JniDBFactory(); - String path = conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH); - File p = new File(path); - if (!p.exists()) { - if (!p.mkdirs()) { - throw new IOException("Couldn't create directory for leveldb " + - "timeline store " + path); + Path dbPath = new Path( + conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH), FILENAME); + FileSystem localFS = null; + try { + localFS = FileSystem.getLocal(conf); + if (!localFS.exists(dbPath)) { + if (!localFS.mkdirs(dbPath)) { + throw new IOException("Couldn't create directory for leveldb " + + "timeline store " + dbPath); + } + localFS.setPermission(dbPath, LEVELDB_DIR_UMASK); } + } finally { + IOUtils.cleanup(LOG, localFS); } - LOG.info("Using leveldb path " + path); - db = factory.open(new File(path, FILENAME), options); + LOG.info("Using leveldb path " + dbPath); + db = factory.open(new File(dbPath.toString()), options); startTimeWriteCache = Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize( conf))); Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java?rev=1597231&r1=1597230&r2=1597231&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java (original) +++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/TestLeveldbTimelineStore.java Sat May 24 01:24:26 2014 @@ -17,6 +17,10 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.timeline; +import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.File; import java.io.IOException; import java.util.Collections; @@ -28,6 +32,8 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -39,9 +45,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.GenericObjectMapper.writeReverseOrderedLong; -import static org.junit.Assert.assertEquals; - @InterfaceAudience.Private @InterfaceStability.Unstable public class TestLeveldbTimelineStore extends TimelineStoreTestUtils { @@ -51,7 +54,7 @@ public class TestLeveldbTimelineStore ex @Before public void setup() throws Exception { fsContext = FileContext.getLocalFSFileContext(); - Configuration conf = new Configuration(); + Configuration conf = new YarnConfiguration(); fsPath = new File("target", this.getClass().getSimpleName() + "-tmpDir").getAbsoluteFile(); fsContext.delete(new Path(fsPath.getAbsolutePath()), true); @@ -72,6 +75,15 @@ public class TestLeveldbTimelineStore ex } @Test + public void testRootDirPermission() throws IOException { + FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); + FileStatus file = fs.getFileStatus( + new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME)); + assertNotNull(file); + assertEquals(LeveldbTimelineStore.LEVELDB_DIR_UMASK, file.getPermission()); + } + + @Test public void testGetSingleEntity() throws IOException { super.testGetSingleEntity(); ((LeveldbTimelineStore)store).clearStartTimeCache();