Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B6A3F200B81 for ; Tue, 13 Sep 2016 16:45:53 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id B542E160AD2; Tue, 13 Sep 2016 14:45:53 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D4393160AAA for ; Tue, 13 Sep 2016 16:45:52 +0200 (CEST) Received: (qmail 6364 invoked by uid 500); 13 Sep 2016 14:45:51 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 6355 invoked by uid 99); 13 Sep 2016 14:45:51 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Sep 2016 14:45:51 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 96299E00D6; Tue, 13 Sep 2016 14:45:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jlowe@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5630. NM fails to start after downgrade from 2.8 to 2.7. Contributed by Jason Lowe (cherry picked from commit e7933097354a246b080b46f1a4ca2ef0f39f3b38) Date: Tue, 13 Sep 2016 14:45:51 +0000 (UTC) archived-at: Tue, 13 Sep 2016 14:45:53 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 c06cf0f89 -> 1ac8206a8 YARN-5630. NM fails to start after downgrade from 2.8 to 2.7. Contributed by Jason Lowe (cherry picked from commit e7933097354a246b080b46f1a4ca2ef0f39f3b38) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ac8206a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ac8206a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ac8206a Branch: refs/heads/branch-2.8 Commit: 1ac8206a85fac4293611fd390938a3c90c10bef8 Parents: c06cf0f Author: Jason Lowe Authored: Tue Sep 13 14:41:27 2016 +0000 Committer: Jason Lowe Committed: Tue Sep 13 14:43:55 2016 +0000 ---------------------------------------------------------------------- .../recovery/NMLeveldbStateStoreService.java | 23 ++++-- .../TestNMLeveldbStateStoreService.java | 84 +++++++++++--------- 2 files changed, 65 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ac8206a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index 3d23334..f5a36f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -260,21 +260,24 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { @Override public void storeContainer(ContainerId containerId, int containerVersion, StartContainerRequest startRequest) throws IOException { + String idStr = containerId.toString(); if (LOG.isDebugEnabled()) { - LOG.debug("storeContainer: containerId= " + containerId + LOG.debug("storeContainer: containerId= " + idStr + ", startRequest= " + startRequest); } - String keyRequest = CONTAINERS_KEY_PREFIX + containerId.toString() + String keyRequest = CONTAINERS_KEY_PREFIX + idStr + CONTAINER_REQUEST_KEY_SUFFIX; - String keyVersion = CONTAINERS_KEY_PREFIX + containerId.toString() - + CONTAINER_VERSION_KEY_SUFFIX; + String keyVersion = getContainerVersionKey(idStr); try { WriteBatch batch = db.createWriteBatch(); try { batch.put(bytes(keyRequest), ((StartContainerRequestPBImpl) startRequest) .getProto().toByteArray()); - batch.put(bytes(keyVersion), bytes(Integer.toString(containerVersion))); + if (containerVersion != 0) { + batch.put(bytes(keyVersion), + bytes(Integer.toString(containerVersion))); + } db.write(batch); } finally { batch.close(); @@ -284,6 +287,11 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { } } + @VisibleForTesting + String getContainerVersionKey(String containerId) { + return CONTAINERS_KEY_PREFIX + containerId + CONTAINER_VERSION_KEY_SUFFIX; + } + @Override public void storeContainerDiagnostics(ContainerId containerId, StringBuilder diagnostics) throws IOException { @@ -1123,6 +1131,11 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { return CURRENT_VERSION_INFO; } + @VisibleForTesting + DB getDB() { + return db; + } + /** * 1) Versioning scheme: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc. * 2) Any incompatible change of state-store is a major upgrade, and any http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ac8206a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index 6755ec0..7faac37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.recovery; +import static org.fusesource.leveldbjni.JniDBFactory.bytes; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -78,7 +79,6 @@ import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.iq80.leveldb.DB; import org.junit.After; import org.junit.Assert; @@ -226,48 +226,21 @@ public class TestNMLeveldbStateStoreService { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 4); ContainerId containerId = ContainerId.newContainerId(appAttemptId, 5); - LocalResource lrsrc = LocalResource.newInstance( - URL.newInstance("hdfs", "somehost", 12345, "/some/path/to/rsrc"), - LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 123L, - 1234567890L); - Map localResources = - new HashMap(); - localResources.put("rsrc", lrsrc); - Map env = new HashMap(); - env.put("somevar", "someval"); - List containerCmds = new ArrayList(); - containerCmds.add("somecmd"); - containerCmds.add("somearg"); - Map serviceData = new HashMap(); - serviceData.put("someservice", - ByteBuffer.wrap(new byte[] { 0x1, 0x2, 0x3 })); - ByteBuffer containerTokens = - ByteBuffer.wrap(new byte[] { 0x7, 0x8, 0x9, 0xa }); - Map acls = - new HashMap(); - acls.put(ApplicationAccessType.VIEW_APP, "viewuser"); - acls.put(ApplicationAccessType.MODIFY_APP, "moduser"); - ContainerLaunchContext clc = ContainerLaunchContext.newInstance( - localResources, env, containerCmds, serviceData, containerTokens, - acls); - Resource containerRsrc = Resource.newInstance(1357, 3); - ContainerTokenIdentifier containerTokenId = - new ContainerTokenIdentifier(containerId, "host", "user", - containerRsrc, 9876543210L, 42, 2468, Priority.newInstance(7), - 13579); - Token containerToken = Token.newInstance(containerTokenId.getBytes(), - ContainerTokenIdentifier.KIND.toString(), "password".getBytes(), - "tokenservice"); - StartContainerRequest containerReq = - StartContainerRequest.newInstance(clc, containerToken); + StartContainerRequest containerReq = createContainerRequest(containerId); // store a container and verify recovered - stateStore.storeContainer(containerId, 1, containerReq); + stateStore.storeContainer(containerId, 0, containerReq); + + // verify the container version key is not stored for new containers + DB db = stateStore.getDB(); + assertNull("version key present for new container", db.get(bytes( + stateStore.getContainerVersionKey(containerId.toString())))); + restartStateStore(); recoveredContainers = stateStore.loadContainersState(); assertEquals(1, recoveredContainers.size()); RecoveredContainerState rcs = recoveredContainers.get(0); - assertEquals(1, rcs.getVersion()); + assertEquals(0, rcs.getVersion()); assertEquals(RecoveredContainerStatus.REQUESTED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); assertEquals(false, rcs.getKilled()); @@ -344,6 +317,43 @@ public class TestNMLeveldbStateStoreService { assertTrue(recoveredContainers.isEmpty()); } + private StartContainerRequest createContainerRequest( + ContainerId containerId) { + LocalResource lrsrc = LocalResource.newInstance( + URL.newInstance("hdfs", "somehost", 12345, "/some/path/to/rsrc"), + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 123L, + 1234567890L); + Map localResources = + new HashMap(); + localResources.put("rsrc", lrsrc); + Map env = new HashMap(); + env.put("somevar", "someval"); + List containerCmds = new ArrayList(); + containerCmds.add("somecmd"); + containerCmds.add("somearg"); + Map serviceData = new HashMap(); + serviceData.put("someservice", + ByteBuffer.wrap(new byte[] { 0x1, 0x2, 0x3 })); + ByteBuffer containerTokens = + ByteBuffer.wrap(new byte[] { 0x7, 0x8, 0x9, 0xa }); + Map acls = + new HashMap(); + acls.put(ApplicationAccessType.VIEW_APP, "viewuser"); + acls.put(ApplicationAccessType.MODIFY_APP, "moduser"); + ContainerLaunchContext clc = ContainerLaunchContext.newInstance( + localResources, env, containerCmds, serviceData, containerTokens, + acls); + Resource containerRsrc = Resource.newInstance(1357, 3); + ContainerTokenIdentifier containerTokenId = + new ContainerTokenIdentifier(containerId, "host", "user", + containerRsrc, 9876543210L, 42, 2468, Priority.newInstance(7), + 13579); + Token containerToken = Token.newInstance(containerTokenId.getBytes(), + ContainerTokenIdentifier.KIND.toString(), "password".getBytes(), + "tokenservice"); + return StartContainerRequest.newInstance(clc, containerToken); + } + @Test public void testStartResourceLocalization() throws IOException { String user = "somebody"; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org