hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject [01/15] hadoop git commit: YARN-6903. Yarn-native-service framework core rewrite. Contributed by Jian He
Date Mon, 14 Aug 2017 21:13:39 GMT
Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services f1a358e17 -> 164c0c4c9


http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
new file mode 100644
index 0000000..1a1593a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+/**
+ * A {@link CharSequence} appender that considers its {@link #limit} as upper
+ * bound.
+ * <p>
+ * When {@link #limit} would be reached on append, past messages will be
+ * truncated from head, and a header telling the user about truncation will be
+ * prepended, with ellipses in between header and messages.
+ * <p>
+ * Note that header and ellipses are not counted against {@link #limit}.
+ * <p>
+ * An example:
+ *
+ * <pre>
+ * {@code
+ *   // At the beginning it's an empty string
+ *   final Appendable shortAppender = new BoundedAppender(80);
+ *   // The whole message fits into limit
+ *   shortAppender.append(
+ *       "message1 this is a very long message but fitting into limit\n");
+ *   // The first message is truncated, the second not
+ *   shortAppender.append("message2 this is shorter than the previous one\n");
+ *   // The first message is deleted, the second truncated, the third
+ *   // preserved
+ *   shortAppender.append("message3 this is even shorter message, maybe.\n");
+ *   // The first two are deleted, the third one truncated, the last preserved
+ *   shortAppender.append("message4 the shortest one, yet the greatest :)");
+ *   // Current contents are like this:
+ *   // Diagnostic messages truncated, showing last 80 chars out of 199:
+ *   // ...s is even shorter message, maybe.
+ *   // message4 the shortest one, yet the greatest :)
+ * }
+ * </pre>
+ * <p>
+ * Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed
+ * just like in {@link StringBuilder#append(CharSequence) original
+ * implementation}.
+ * <p>
+ * Note that this class is not thread safe.
+ */
+@VisibleForTesting
+public class BoundedAppender {
+  @VisibleForTesting
+  public static final String TRUNCATED_MESSAGES_TEMPLATE =
+      "Diagnostic messages truncated, showing last "
+          + "%d chars out of %d:%n...%s";
+
+  private final int limit;
+  private final StringBuilder messages = new StringBuilder();
+  private int totalCharacterCount = 0;
+
+  public BoundedAppender(final int limit) {
+    Preconditions.checkArgument(limit > 0, "limit should be positive");
+
+    this.limit = limit;
+  }
+
+  /**
+   * Append a {@link CharSequence} considering {@link #limit}, truncating
+   * from the head of {@code csq} or {@link #messages} when necessary.
+   *
+   * @param csq the {@link CharSequence} to append
+   * @return this
+   */
+  public BoundedAppender append(final CharSequence csq) {
+    appendAndCount(csq);
+    checkAndCut();
+
+    return this;
+  }
+
+  private void appendAndCount(final CharSequence csq) {
+    final int before = messages.length();
+    messages.append(csq);
+    final int after = messages.length();
+    totalCharacterCount += after - before;
+  }
+
+  private void checkAndCut() {
+    if (messages.length() > limit) {
+      final int newStart = messages.length() - limit;
+      messages.delete(0, newStart);
+    }
+  }
+
+  /**
+   * Get current length of messages considering truncates
+   * without header and ellipses.
+   *
+   * @return current length
+   */
+  public int length() {
+    return messages.length();
+  }
+
+  /**
+   * Get a string representation of the actual contents, displaying also a
+   * header and ellipses when there was a truncate.
+   *
+   * @return String representation of the {@link #messages}
+   */
+  @Override
+  public String toString() {
+    if (messages.length() < totalCharacterCount) {
+      return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(),
+          totalCharacterCount, messages.toString());
+    }
+
+    return messages.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java
new file mode 100644
index 0000000..2b9cfce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestBoundedAppender.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test class for {@link BoundedAppender}.
+ */
+public class TestBoundedAppender {
+  @Rule
+  public ExpectedException expected = ExpectedException.none();
+
+  @Test
+  public void initWithZeroLimitThrowsException() {
+    expected.expect(IllegalArgumentException.class);
+    expected.expectMessage("limit should be positive");
+
+    new BoundedAppender(0);
+  }
+
+  @Test
+  public void nullAppendedNullStringRead() {
+    final BoundedAppender boundedAppender = new BoundedAppender(4);
+    boundedAppender.append(null);
+
+    assertEquals("null appended, \"null\" read", "null",
+        boundedAppender.toString());
+  }
+
+  @Test
+  public void appendBelowLimitOnceValueIsReadCorrectly() {
+    final BoundedAppender boundedAppender = new BoundedAppender(2);
+
+    boundedAppender.append("ab");
+
+    assertEquals("value appended is read correctly", "ab",
+        boundedAppender.toString());
+  }
+
+  @Test
+  public void appendValuesBelowLimitAreReadCorrectlyInFifoOrder() {
+    final BoundedAppender boundedAppender = new BoundedAppender(3);
+
+    boundedAppender.append("ab");
+    boundedAppender.append("cd");
+    boundedAppender.append("e");
+    boundedAppender.append("fg");
+
+    assertEquals("last values appended fitting limit are read correctly",
+        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"),
+        boundedAppender.toString());
+  }
+
+  @Test
+  public void appendLastAboveLimitPreservesLastMessagePostfix() {
+    final BoundedAppender boundedAppender = new BoundedAppender(3);
+
+    boundedAppender.append("ab");
+    boundedAppender.append("cde");
+    boundedAppender.append("fghij");
+
+    assertEquals(
+        "last value appended above limit postfix is read correctly", String
+            .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 10, "hij"),
+        boundedAppender.toString());
+  }
+
+  @Test
+  public void appendMiddleAboveLimitPreservesLastMessageAndMiddlePostfix() {
+    final BoundedAppender boundedAppender = new BoundedAppender(3);
+
+    boundedAppender.append("ab");
+    boundedAppender.append("cde");
+
+    assertEquals("last value appended above limit postfix is read correctly",
+        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 5, "cde"),
+        boundedAppender.toString());
+
+    boundedAppender.append("fg");
+
+    assertEquals(
+        "middle value appended above limit postfix and last value are "
+            + "read correctly",
+        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"),
+        boundedAppender.toString());
+
+    boundedAppender.append("hijkl");
+
+    assertEquals(
+        "last value appended above limit postfix is read correctly", String
+            .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 12, "jkl"),
+        boundedAppender.toString());
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
index 443654d..704b097 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryOperationsFactory.java
@@ -67,6 +67,14 @@ public final class RegistryOperationsFactory {
     return operations;
   }
 
+  public static RegistryOperationsClient createClient(String name,
+      Configuration conf) {
+    Preconditions.checkArgument(conf != null, "Null configuration");
+    RegistryOperationsClient operations = new RegistryOperationsClient(name);
+    operations.init(conf);
+    return operations;
+  }
+
   /**
    * Create and initialize an anonymous read/write registry operations instance.
    * In a secure cluster, this instance will only have read access to the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index ad008c4..8713920 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -261,10 +261,9 @@ public class CuratorService extends CompositeService
     int retryCeiling = conf.getInt(KEY_REGISTRY_ZK_RETRY_CEILING,
         DEFAULT_ZK_RETRY_CEILING);
 
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Creating CuratorService with connection {}",
+    LOG.info("Creating CuratorService with connection {}",
           connectionDescription);
-    }
+
     CuratorFramework framework;
 
     synchronized (CuratorService.class) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
index 271ab25..4c911da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistryOperationsService.java
@@ -107,8 +107,10 @@ public class RegistryOperationsService extends CuratorService
     validatePath(path);
     // validate the record before putting it
     RegistryTypeUtils.validateServiceRecord(path, record);
-    LOG.info("Bound at {} : {}", path, record);
 
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Bound at {} : ServiceRecord = {}", path, record);
+    }
     CreateMode mode = CreateMode.PERSISTENT;
     byte[] bytes = serviceRecordMarshal.toBytes(record);
     zkSet(path, mode, bytes, getClientAcls(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 368832a..bcd1a9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
-import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.security.AccessRequest;
@@ -65,7 +64,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 
@@ -349,36 +347,6 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
         createAndPopulateNewRMApp(appContext, appState.getSubmitTime(),
             appState.getUser(), true, appState.getStartTime());
 
-    // If null amReq has been returned, check if it is the case that
-    // application has specified node label expression while node label
-    // has been disabled. Reject the recovery of this application if it
-    // is true and give clear message so that user can react properly.
-    if (!appContext.getUnmanagedAM() &&
-        (application.getAMResourceRequests() == null ||
-            application.getAMResourceRequests().isEmpty()) &&
-        !YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
-      // check application submission context and see if am resource request
-      // or application itself contains any node label expression.
-      List<ResourceRequest> amReqsFromAppContext =
-          appContext.getAMContainerResourceRequests();
-      String labelExp =
-          (amReqsFromAppContext != null && !amReqsFromAppContext.isEmpty()) ?
-          amReqsFromAppContext.get(0).getNodeLabelExpression() : null;
-      if (labelExp == null) {
-        labelExp = appContext.getNodeLabelExpression();
-      }
-      if (labelExp != null &&
-          !labelExp.equals(RMNodeLabelsManager.NO_LABEL)) {
-        String message = "Failed to recover application " + appId
-            + ". NodeLabel is not enabled in cluster, but AM resource request "
-            + "contains a label expression.";
-        LOG.warn(message);
-        application.handle(
-            new RMAppEvent(appId, RMAppEventType.APP_REJECTED, message));
-        return;
-      }
-    }
-
     application.handle(new RMAppRecoverEvent(appId, rmState));
   }
 
@@ -398,28 +366,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
     }
 
     ApplicationId applicationId = submissionContext.getApplicationId();
-    List<ResourceRequest> amReqs = null;
-    try {
-      amReqs = validateAndCreateResourceRequest(submissionContext, isRecovery);
-    } catch (InvalidLabelResourceRequestException e) {
-      // This can happen if the application had been submitted and run
-      // with Node Label enabled but recover with Node Label disabled.
-      // Thus there might be node label expression in the application's
-      // resource requests. If this is the case, create RmAppImpl with
-      // null amReq and reject the application later with clear error
-      // message. So that the application can still be tracked by RM
-      // after recovery and user can see what's going on and react accordingly.
-      if (isRecovery &&
-          !YarnConfiguration.areNodeLabelsEnabled(this.conf)) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("AMResourceRequest is not created for " + applicationId
-              + ". NodeLabel is not enabled in cluster, but AM resource "
-              + "request contains a label expression.");
-        }
-      } else {
-        throw e;
-      }
-    }
+    List<ResourceRequest> amReqs = validateAndCreateResourceRequest(
+        submissionContext, isRecovery);
 
     // Verify and get the update application priority and set back to
     // submissionContext

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java
index f10e885..e748955 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/blacklist/SimpleBlacklistManager.java
@@ -18,15 +18,15 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.blacklist;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
+
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
-
 /**
  * Maintains a list of failed nodes and returns that as long as number of
  * blacklisted nodes is below a threshold percentage of total nodes. If more

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 4210c54..c00f505 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -38,7 +38,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 import javax.crypto.SecretKey;
 
-import com.google.common.base.Preconditions;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -110,6 +109,8 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.util.BoundedAppender;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -1552,38 +1553,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable
{
     }
   }
 
-  private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
-    switch (exitStatus) {
-    case ContainerExitStatus.PREEMPTED:
-    case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER:
-    case ContainerExitStatus.KILLED_BY_APPMASTER:
-    case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION:
-    case ContainerExitStatus.ABORTED:
-      // Neither the app's fault nor the system's fault. This happens by design,
-      // so no need for skipping nodes
-      return false;
-    case ContainerExitStatus.DISKS_FAILED:
-      // This container is marked with this exit-status means that the node is
-      // already marked as unhealthy given that most of the disks failed. So, no
-      // need for any explicit skipping of nodes.
-      return false;
-    case ContainerExitStatus.KILLED_EXCEEDED_VMEM:
-    case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
-      // No point in skipping the node as it's not the system's fault
-      return false;
-    case ContainerExitStatus.SUCCESS:
-      return false;
-    case ContainerExitStatus.INVALID:
-      // Ideally, this shouldn't be considered for skipping a node. But in
-      // reality, it seems like there are cases where we are not setting
-      // exit-code correctly and so it's better to be conservative. See
-      // YARN-4284.
-      return true;
-    default:
-      return true;
-    }
-  }
-
   private static final class UnmanagedAMAttemptSavedTransition
                                                 extends AMLaunchedTransition {
     @Override
@@ -1963,7 +1932,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
         containerFinishedEvent.getContainerStatus();
     if (containerStatus != null) {
       int exitStatus = containerStatus.getExitStatus();
-      if (shouldCountTowardsNodeBlacklisting(exitStatus)) {
+      if (Apps.shouldCountTowardsNodeBlacklisting(exitStatus)) {
         appAttempt.addAMNodeToBlackList(nodeId);
       }
     } else {
@@ -2279,114 +2248,4 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable
{
     return Collections.EMPTY_SET;
   }
 
-  /**
-   * A {@link CharSequence} appender that considers its {@link #limit} as upper
-   * bound.
-   * <p>
-   * When {@link #limit} would be reached on append, past messages will be
-   * truncated from head, and a header telling the user about truncation will be
-   * prepended, with ellipses in between header and messages.
-   * <p>
-   * Note that header and ellipses are not counted against {@link #limit}.
-   * <p>
-   * An example:
-   *
-   * <pre>
-   * {@code
-   *   // At the beginning it's an empty string
-   *   final Appendable shortAppender = new BoundedAppender(80);
-   *   // The whole message fits into limit
-   *   shortAppender.append(
-   *       "message1 this is a very long message but fitting into limit\n");
-   *   // The first message is truncated, the second not
-   *   shortAppender.append("message2 this is shorter than the previous one\n");
-   *   // The first message is deleted, the second truncated, the third
-   *   // preserved
-   *   shortAppender.append("message3 this is even shorter message, maybe.\n");
-   *   // The first two are deleted, the third one truncated, the last preserved
-   *   shortAppender.append("message4 the shortest one, yet the greatest :)");
-   *   // Current contents are like this:
-   *   // Diagnostic messages truncated, showing last 80 chars out of 199:
-   *   // ...s is even shorter message, maybe.
-   *   // message4 the shortest one, yet the greatest :)
-   * }
-   * </pre>
-   * <p>
-   * Note that <tt>null</tt> values are {@link #append(CharSequence) append}ed
-   * just like in {@link StringBuilder#append(CharSequence) original
-   * implementation}.
-   * <p>
-   * Note that this class is not thread safe.
-   */
-  @VisibleForTesting
-  static class BoundedAppender {
-    @VisibleForTesting
-    static final String TRUNCATED_MESSAGES_TEMPLATE =
-        "Diagnostic messages truncated, showing last "
-            + "%d chars out of %d:%n...%s";
-
-    private final int limit;
-    private final StringBuilder messages = new StringBuilder();
-    private int totalCharacterCount = 0;
-
-    BoundedAppender(final int limit) {
-      Preconditions.checkArgument(limit > 0, "limit should be positive");
-
-      this.limit = limit;
-    }
-
-    /**
-     * Append a {@link CharSequence} considering {@link #limit}, truncating
-     * from the head of {@code csq} or {@link #messages} when necessary.
-     *
-     * @param csq the {@link CharSequence} to append
-     * @return this
-     */
-    BoundedAppender append(final CharSequence csq) {
-      appendAndCount(csq);
-      checkAndCut();
-
-      return this;
-    }
-
-    private void appendAndCount(final CharSequence csq) {
-      final int before = messages.length();
-      messages.append(csq);
-      final int after = messages.length();
-      totalCharacterCount += after - before;
-    }
-
-    private void checkAndCut() {
-      if (messages.length() > limit) {
-        final int newStart = messages.length() - limit;
-        messages.delete(0, newStart);
-      }
-    }
-
-    /**
-     * Get current length of messages considering truncates
-     * without header and ellipses.
-     *
-     * @return current length
-     */
-    int length() {
-      return messages.length();
-    }
-
-    /**
-     * Get a string representation of the actual contents, displaying also a
-     * header and ellipses when there was a truncate.
-     *
-     * @return String representation of the {@link #messages}
-     */
-    @Override
-    public String toString() {
-      if (messages.length() < totalCharacterCount) {
-        return String.format(TRUNCATED_MESSAGES_TEMPLATE, messages.length(),
-            totalCharacterCount, messages.toString());
-      }
-
-      return messages.toString();
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index c00b7be..b40ce40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstant
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -518,12 +519,20 @@ public abstract class AbstractYarnScheduler
     container.setVersion(status.getVersion());
     ApplicationAttemptId attemptId =
         container.getId().getApplicationAttemptId();
+    String labelExpression = status.getNodeLabelExpression();
+    // If NodeLabel is disabled but recovered container has label expression
+    // its better to suppress that and considered as default label.
+    if (!status.getNodeLabelExpression().isEmpty() && !YarnConfiguration
+        .areNodeLabelsEnabled(rmContext.getYarnConfiguration())) {
+      labelExpression = RMNodeLabelsManager.NO_LABEL;
+    }
+
     RMContainer rmContainer =
         new RMContainerImpl(container,
             SchedulerRequestKey.extractFrom(container), attemptId,
             node.getNodeID(), applications.get(
             attemptId.getApplicationId()).getUser(), rmContext,
-            status.getCreationTime(), status.getNodeLabelExpression());
+            status.getCreationTime(), labelExpression);
     return rmContainer;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c67f1ce..7b554db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -22,6 +22,8 @@ import java.util.Set;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
@@ -55,7 +57,9 @@ import org.apache.hadoop.yarn.util.resource.Resources;
 @Private
 @Unstable
 public class SchedulerUtils {
-  
+
+  private static final Log LOG = LogFactory.getLog(SchedulerUtils.class);
+
   private static final RecordFactory recordFactory = 
       RecordFactoryProvider.getRecordFactory(null);
 
@@ -200,9 +204,14 @@ public class SchedulerUtils {
       String labelExp = resReq.getNodeLabelExpression();
       if (!(RMNodeLabelsManager.NO_LABEL.equals(labelExp)
           || null == labelExp)) {
-        throw new InvalidLabelResourceRequestException(
-            "Invalid resource request, node label not enabled "
-                + "but request contains label expression");
+        String message = "NodeLabel is not enabled in cluster, but resource"
+            + " request contains a label expression.";
+        LOG.warn(message);
+        if (!isRecovery) {
+          throw new InvalidLabelResourceRequestException(
+              "Invalid resource request, node label not enabled "
+                  + "but request contains label expression");
+        }
       }
     }
     if (null == queueInfo) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 955b4b6..475c44e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -2589,14 +2589,14 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase
{
       }
     };
 
-    // rm should successfully start with app1 loaded back in FAILED state
-    // due to node label not enabled but am resource request contains
-    // node label expression.
+    // rm should successfully start with app1 loaded back in SUCCESS state
+    // by pushing app to run default label for am container and let other
+    // containers to run normally.
+
     try {
       rm2.start();
       Assert.assertTrue("RM start successfully", true);
       Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
-      rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
     } catch (Exception e) {
       LOG.debug("Exception on start", e);
       Assert.fail("RM should start without any issue");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java
deleted file mode 100644
index 9cb1e04..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestBoundedAppender.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import static org.junit.Assert.assertEquals;
-import static org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl.BoundedAppender;
-
-/**
- * Test class for {@link BoundedAppender}.
- */
-public class TestBoundedAppender {
-  @Rule
-  public ExpectedException expected = ExpectedException.none();
-
-  @Test
-  public void initWithZeroLimitThrowsException() {
-    expected.expect(IllegalArgumentException.class);
-    expected.expectMessage("limit should be positive");
-
-    new BoundedAppender(0);
-  }
-
-  @Test
-  public void nullAppendedNullStringRead() {
-    final BoundedAppender boundedAppender = new BoundedAppender(4);
-    boundedAppender.append(null);
-
-    assertEquals("null appended, \"null\" read", "null",
-        boundedAppender.toString());
-  }
-
-  @Test
-  public void appendBelowLimitOnceValueIsReadCorrectly() {
-    final BoundedAppender boundedAppender = new BoundedAppender(2);
-
-    boundedAppender.append("ab");
-
-    assertEquals("value appended is read correctly", "ab",
-        boundedAppender.toString());
-  }
-
-  @Test
-  public void appendValuesBelowLimitAreReadCorrectlyInFifoOrder() {
-    final BoundedAppender boundedAppender = new BoundedAppender(3);
-
-    boundedAppender.append("ab");
-    boundedAppender.append("cd");
-    boundedAppender.append("e");
-    boundedAppender.append("fg");
-
-    assertEquals("last values appended fitting limit are read correctly",
-        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"),
-        boundedAppender.toString());
-  }
-
-  @Test
-  public void appendLastAboveLimitPreservesLastMessagePostfix() {
-    final BoundedAppender boundedAppender = new BoundedAppender(3);
-
-    boundedAppender.append("ab");
-    boundedAppender.append("cde");
-    boundedAppender.append("fghij");
-
-    assertEquals(
-        "last value appended above limit postfix is read correctly", String
-            .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 10, "hij"),
-        boundedAppender.toString());
-  }
-
-  @Test
-  public void appendMiddleAboveLimitPreservesLastMessageAndMiddlePostfix() {
-    final BoundedAppender boundedAppender = new BoundedAppender(3);
-
-    boundedAppender.append("ab");
-    boundedAppender.append("cde");
-
-    assertEquals("last value appended above limit postfix is read correctly",
-        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 5, "cde"),
-        boundedAppender.toString());
-
-    boundedAppender.append("fg");
-
-    assertEquals(
-        "middle value appended above limit postfix and last value are "
-            + "read correctly",
-        String.format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 7, "efg"),
-        boundedAppender.toString());
-
-    boundedAppender.append("hijkl");
-
-    assertEquals(
-        "last value appended above limit postfix is read correctly", String
-            .format(BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 3, 12, "jkl"),
-        boundedAppender.toString());
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
index 19b5dd9..295b59f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImplDiagnostics.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.util.BoundedAppender;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -86,7 +87,7 @@ public class TestRMAppAttemptImplDiagnostics {
     appAttempt.appendDiagnostics(beyondLimit);
 
     final String truncated = String.format(
-        RMAppAttemptImpl.BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024,
+        BoundedAppender.TRUNCATED_MESSAGES_TEMPLATE, 1024,
         1025, beyondLimit.substring(1));
 
     assertEquals("messages beyond limit should be truncated", truncated,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message