hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [36/50] [abbrv] hadoop git commit: YARN-6354. LeveldbRMStateStore can parse invalid keys when recovering reservations. Contributed by Jason Lowe
Date Tue, 04 Apr 2017 19:57:00 GMT
YARN-6354. LeveldbRMStateStore can parse invalid keys when recovering reservations. Contributed
by Jason Lowe


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/318bfb01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/318bfb01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/318bfb01

Branch: refs/heads/HDFS-7240
Commit: 318bfb01bc6793da09e32e9cc292eb63224b6ca2
Parents: 4d1fac5
Author: Eric Payne <epayne@apache.org>
Authored: Fri Mar 31 12:30:35 2017 -0500
Committer: Eric Payne <epayne@apache.org>
Committed: Fri Mar 31 12:30:35 2017 -0500

----------------------------------------------------------------------
 .../recovery/LeveldbRMStateStore.java            |  8 ++++++++
 .../recovery/TestLeveldbRMStateStore.java        | 19 +++++++++++++++----
 2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
index 02f90dd..2ca53db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java
@@ -214,6 +214,11 @@ public class LeveldbRMStateStore extends RMStateStore {
     return db == null;
   }
 
+  @VisibleForTesting
+  DB getDatabase() {
+    return db;
+  }
+
   @Override
   protected Version loadVersion() throws Exception {
     Version version = null;
@@ -284,6 +289,9 @@ public class LeveldbRMStateStore extends RMStateStore {
       while (iter.hasNext()) {
         Entry<byte[],byte[]> entry = iter.next();
         String key = asString(entry.getKey());
+        if (!key.startsWith(RM_RESERVATION_KEY_PREFIX)) {
+          break;
+        }
 
         String planReservationString =
             key.substring(RM_RESERVATION_KEY_PREFIX.length());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/318bfb01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
index 4297e73..51adbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.records.Version;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.fusesource.leveldbjni.JniDBFactory;
 import org.iq80.leveldb.DB;
 import org.junit.After;
 import org.junit.Before;
@@ -125,17 +126,27 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase {
   public void testCompactionCycle() throws Exception {
     final DB mockdb = mock(DB.class);
     conf.setLong(YarnConfiguration.RM_LEVELDB_COMPACTION_INTERVAL_SECS, 1);
-    LeveldbRMStateStore store = new LeveldbRMStateStore() {
+    stateStore = new LeveldbRMStateStore() {
       @Override
       protected DB openDatabase() throws Exception {
         return mockdb;
       }
     };
-    store.init(conf);
-    store.start();
+    stateStore.init(conf);
+    stateStore.start();
     verify(mockdb, timeout(10000)).compactRange(
         (byte[]) isNull(), (byte[]) isNull());
-    store.close();
+  }
+
+  @Test
+  public void testBadKeyIteration() throws Exception {
+    stateStore = new LeveldbRMStateStore();
+    stateStore.init(conf);
+    stateStore.start();
+    DB db = stateStore.getDatabase();
+    // add an entry that appears at the end of the database when iterating
+    db.put(JniDBFactory.bytes("zzz"), JniDBFactory.bytes("z"));
+    stateStore.loadState();
   }
 
   class LeveldbStateStoreTester implements RMStateStoreHelper {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message