hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [10/11] hadoop git commit: YARN-6335. Port slider's groovy unit tests to yarn native services. Contributed by Billie Rinaldi
Date Thu, 20 Apr 2017 15:53:39 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
index 18aa1f5..eb87108 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
@@ -26,7 +26,7 @@ import java.util.List;
 
 @Parameters(commandNames = { SliderActions.ACTION_UPGRADE },
             commandDescription = SliderActions.DESCRIBE_ACTION_UPGRADE)
-public class ActionUpgradeArgs extends AbstractActionArgs {
+public class ActionUpgradeArgs extends AbstractClusterBuildingActionArgs {
 
   @Override
   public String getActionName() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
deleted file mode 100644
index f171708..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Delegate for application and resource options
- */
-public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
-
-
-  /**
-   * Options key value
-   */
-  @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
-             description = ARG_OPTION + "<name> <value>",
-             splitter = DontSplitArguments.class)
-  public List<String> optionTuples = new ArrayList<>(0);
-
-
-  /**
-   * All the app component option triples
-   */
-  @Parameter(names = {ARG_COMP_OPT,  ARG_COMP_OPT_SHORT,  ARG_ROLEOPT}, arity = 3,
-             description = "Component option " + ARG_COMP_OPT +
-                           " <component> <name> <option>",
-             splitter = DontSplitArguments.class)
-  public List<String> compOptTriples = new ArrayList<>(0);
-
-  /**
-   * Resource Options
-   */
-  @Parameter(names = {ARG_RESOURCE_OPT, ARG_RESOURCE_OPT_SHORT}, arity = 2,
-             description = "Resource option "+ ARG_RESOURCE_OPT + "<name> <value>",
-             splitter = DontSplitArguments.class)
-  public List<String> resOptionTuples = new ArrayList<>(0);
-
-
-  /**
-   * All the resource component option triples
-   */
-  @Parameter(names = {ARG_RES_COMP_OPT, ARG_RES_COMP_OPT_SHORT,}, arity = 3,
-             description = "Component resource option " + ARG_RES_COMP_OPT +
-                           " <component> <name> <option>",
-             splitter = DontSplitArguments.class)
-  public List<String> resCompOptTriples = new ArrayList<>(0);
-
-
-  public Map<String, String> getOptionsMap() throws
-                                             BadCommandArgumentsException {
-    return convertTupleListToMap(ARG_OPTION, optionTuples);
-  }
-
-  /**
-   * Get the role heap mapping (may be empty, but never null)
-   * @return role heap mapping
-   * @throws BadCommandArgumentsException parse problem
-   */
-  public Map<String, Map<String, String>> getCompOptionMap() throws
-                                                             BadCommandArgumentsException {
-    return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
-  }
-
-  public Map<String, String> getResourceOptionsMap() throws
-                                             BadCommandArgumentsException {
-    return convertTupleListToMap(ARG_RESOURCE_OPT, resOptionTuples);
-  }
-
-  /**
-   * Get the role heap mapping (may be empty, but never null)
-   * @return role heap mapping
-   * @throws BadCommandArgumentsException parse problem
-   */
-  public Map<String, Map<String, String>> getResourceCompOptionMap() throws
-                                                             BadCommandArgumentsException {
-    return convertTripleListToMaps(ARG_RES_COMP_OPT, resCompOptTriples);
-  }
-
-  public void setOption(String key, String value) {
-    optionTuples.add(key);
-    optionTuples.add(value);
-  }
-
-  public void setResourceOption(String key, String value) {
-    resOptionTuples.add(key);
-    resOptionTuples.add(value);
-  }
-  
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 45c1fbd..e978957 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -107,12 +107,7 @@ public interface Arguments {
   String ARG_LIFETIME = "--lifetime";
   String ARG_REPLACE_PKG = "--replacepkg";
   String ARG_RESOURCE = "--resource";
-  String ARG_RESOURCES = "--resources";
-  String ARG_RES_COMP_OPT = "--rescompopt";
-  String ARG_RES_COMP_OPT_SHORT = "--rco";
   String ARG_RESOURCE_MANAGER = "--rm";
-  String ARG_RESOURCE_OPT = "--resopt";
-  String ARG_RESOURCE_OPT_SHORT = "-ro";
   String ARG_SECURE = "--secure";
   String ARG_SERVICETYPE = "--servicetype";
   String ARG_SERVICES = "--services";
@@ -120,7 +115,6 @@ public interface Arguments {
   String ARG_SOURCE = "--source";
   String ARG_STATE = "--state";
   String ARG_SYSPROP = "-S";
-  String ARG_TEMPLATE = "--template";
   String ARG_TRUSTSTORE = "--truststore";
   String ARG_USER = "--user";
   String ARG_UPLOAD = "--upload";
@@ -136,29 +130,9 @@ public interface Arguments {
  RIGHT PLACE IN THE LIST
  */
 
-
-  /**
-   * Deprecated: use ARG_COMPONENT
-   */
-  @Deprecated
-  String ARG_ROLE = "--role";
-
-  /**
-   * Deprecated: use ARG_COMP_OPT
-   */
-  @Deprecated
-  String ARG_ROLEOPT = "--roleopt";
-
   /**
    * server: URI for the cluster
    */
   String ARG_CLUSTER_URI = "-cluster-uri";
 
-
-  /**
-   * server: Path for the resource manager instance (required)
-   */
-  String ARG_RM_ADDR = "--rm";
-
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index abd2ce7..dbb5a16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -53,6 +53,7 @@ public class ClientArgs extends CommonArgs {
   // =========================================================
 
   private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
+  private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
   private final ActionClientArgs actionClientArgs = new ActionClientArgs();
   private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
   private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
@@ -62,8 +63,6 @@ public class ClientArgs extends CommonArgs {
   private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
   private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
   private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
-  private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
-  private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs();
   private final ActionKDiagArgs actionKDiagArgs = new ActionKDiagArgs();
   private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
   private final ActionKillContainerArgs actionKillContainerArgs =
@@ -71,7 +70,6 @@ public class ClientArgs extends CommonArgs {
   private final ActionListArgs actionListArgs = new ActionListArgs();
   private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
   private final ActionNodesArgs actionNodesArgs = new ActionNodesArgs();
-  private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs();
   private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
   private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
   private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
@@ -95,6 +93,7 @@ public class ClientArgs extends CommonArgs {
 
     addActions(
         actionAMSuicideArgs,
+        actionBuildArgs,
         actionClientArgs,
         actionCreateArgs,
         actionDependencyArgs,
@@ -104,15 +103,12 @@ public class ClientArgs extends CommonArgs {
         actionFlexArgs,
         actionFreezeArgs,
         actionHelpArgs,
-        actionInstallKeytabArgs,
-        actionInstallPackageArgs,
         actionKDiagArgs,
         actionKeytabArgs,
         actionKillContainerArgs,
         actionListArgs,
         actionLookupArgs,
         actionNodesArgs,
-        actionPackageArgs,
         actionRegistryArgs,
         actionResolveArgs,
         actionResourceArgs,
@@ -153,14 +149,12 @@ public class ClientArgs extends CommonArgs {
     return actionAMSuicideArgs;
   }
 
-  public ActionInstallPackageArgs getActionInstallPackageArgs() { return actionInstallPackageArgs; }
+  public ActionBuildArgs getActionBuildArgs() {
+    return actionBuildArgs;
+  }
 
   public ActionClientArgs getActionClientArgs() { return actionClientArgs; }
 
-  public ActionPackageArgs getActionPackageArgs() { return actionPackageArgs; }
-
-  public ActionInstallKeytabArgs getActionInstallKeytabArgs() { return actionInstallKeytabArgs; }
-
   public ActionKDiagArgs getActionKDiagArgs() {
     return actionKDiagArgs;
   }
@@ -250,6 +244,12 @@ public class ClientArgs extends CommonArgs {
       action = ACTION_HELP;
     }
     switch (action) {
+      case ACTION_BUILD:
+        bindCoreAction(actionBuildArgs);
+        //its a builder, so set those actions too
+        buildingActionArgs = actionBuildArgs;
+        break;
+
       case ACTION_CREATE:
         bindCoreAction(actionCreateArgs);
         //its a builder, so set those actions too
@@ -296,14 +296,6 @@ public class ClientArgs extends CommonArgs {
         bindCoreAction(actionHelpArgs);
         break;
 
-      case ACTION_INSTALL_KEYTAB:
-        bindCoreAction(actionInstallKeytabArgs);
-        break;
-
-      case ACTION_INSTALL_PACKAGE:
-        bindCoreAction(actionInstallPackageArgs);
-        break;
-
       case ACTION_KDIAG:
         bindCoreAction(actionKDiagArgs);
         break;
@@ -328,10 +320,6 @@ public class ClientArgs extends CommonArgs {
         bindCoreAction(actionNodesArgs);
         break;
 
-      case ACTION_PACKAGE:
-        bindCoreAction(actionPackageArgs);
-        break;
-
       case ACTION_REGISTRY:
         bindCoreAction(actionRegistryArgs);
         break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
index 5140059..c819b37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -30,7 +30,7 @@ public class ComponentArgsDelegate extends AbstractArgsDelegate {
   /**
    * This is a listing of the roles to create
    */
-  @Parameter(names = {ARG_COMPONENT,  ARG_COMPONENT_SHORT, ARG_ROLE},
+  @Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
              arity = 2,
              description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
              splitter = DontSplitArguments.class)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java
new file mode 100644
index 0000000..e63bd12
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/OptionArgsDelegate.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Delegate for application and resource options.
+ */
+public class OptionArgsDelegate extends AbstractArgsDelegate {
+
+  /**
+   * Options key value.
+   */
+  @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
+             description = ARG_OPTION + "<name> <value>",
+             splitter = DontSplitArguments.class)
+  public List<String> optionTuples = new ArrayList<>(0);
+
+
+  /**
+   * All the app component option triples.
+   */
+  @Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
+             description = "Component option " + ARG_COMP_OPT +
+                           " <component> <name> <option>",
+             splitter = DontSplitArguments.class)
+  public List<String> compOptTriples = new ArrayList<>(0);
+
+  public Map<String, String> getOptionsMap() throws
+                                             BadCommandArgumentsException {
+    return convertTupleListToMap(ARG_OPTION, optionTuples);
+  }
+
+  /**
+   * Get the role heap mapping (may be empty, but never null).
+   * @return role heap mapping
+   * @throws BadCommandArgumentsException parse problem
+   */
+  public Map<String, Map<String, String>> getCompOptionMap()
+      throws BadCommandArgumentsException {
+    return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 82e5903..df1a5fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -39,14 +39,12 @@ public interface SliderActions {
   String ACTION_STOP = "stop";
   String ACTION_HELP = "help";
   String ACTION_INSTALL_KEYTAB = "install-keytab";
-  String ACTION_INSTALL_PACKAGE = "install-package";
   String ACTION_KDIAG = "kdiag";
   String ACTION_KEYTAB = "keytab";
   String ACTION_KILL_CONTAINER = "kill-container";
   String ACTION_LIST = "list";
   String ACTION_LOOKUP = "lookup";
   String ACTION_NODES = "nodes";
-  String ACTION_PACKAGE = "package";
   String ACTION_PREFLIGHT = "preflight";
   String ACTION_RECONFIGURE = "reconfigure";
   String ACTION_REGISTRY = "registry";
@@ -99,12 +97,7 @@ public interface SliderActions {
                         "Start a stopped application";
   String DESCRIBE_ACTION_VERSION =
                         "Print the Slider version information";
-  String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." +
-  		" Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.";
-  String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages";
   String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
-  String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." +
-  		" Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'.";
   String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
   String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
   String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index 02eba49..0e94a29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
 import org.apache.slider.core.exceptions.BadConfigException;
@@ -613,13 +612,6 @@ public class ConfigHelper {
    * Register anything we consider deprecated
    */
   public static void registerDeprecatedConfigItems() {
-    Configuration.addDeprecation(
-        SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
-        RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
-    Configuration.addDeprecation(
-        SliderXmlConfKeys.REGISTRY_PATH,
-        RegistryConstants.KEY_REGISTRY_ZK_ROOT);
-    
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 9d7c40a..80b70b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -21,14 +21,10 @@ package org.apache.slider.common.tools;
 import com.google.common.base.Preconditions;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
-import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
@@ -52,6 +48,8 @@ import org.apache.hadoop.yarn.client.api.AMRMClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.slider.Slider;
 import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
 import org.apache.slider.api.types.ContainerInformation;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
@@ -70,7 +68,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedOutputStream;
-import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -502,6 +499,26 @@ public final class SliderUtils {
   }
 
   /**
+   * Extract the first line of a multi-line string. This is typically used to
+   * prune the stack trace appended to the end of exception messages returned by
+   * YARN in AMRMClientAsync callbacks.
+   *
+   * @param msg
+   *          message string (most likely multi-lines)
+   * @return the first line of a multi-line string or the original string if it
+   *         is a null, empty or single-line
+   */
+  public static String extractFirstLine(String msg) {
+    if (StringUtils.isNotBlank(msg)) {
+      int newlineIndex = msg.indexOf(System.lineSeparator());
+      if (newlineIndex != -1) {
+        msg = msg.substring(0, newlineIndex);
+      }
+    }
+    return msg;
+  }
+
+  /**
    * Create a configuration with Slider-specific tuning.
    * This is done rather than doing custom configs.
    * @return the config
@@ -2046,48 +2063,6 @@ public final class SliderUtils {
     }
   }
 
-  public static InputStream getApplicationResourceInputStream(FileSystem fs,
-      Path appPath,
-      String entry)
-      throws IOException {
-    InputStream is = null;
-    try(FSDataInputStream appStream = fs.open(appPath)) {
-      ZipArchiveInputStream zis = new ZipArchiveInputStream(appStream);
-      ZipArchiveEntry zipEntry;
-      boolean done = false;
-      while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
-        if (entry.equals(zipEntry.getName())) {
-          int size = (int) zipEntry.getSize();
-          if (size != -1) {
-            log.info("Reading {} of size {}", zipEntry.getName(),
-                zipEntry.getSize());
-            byte[] content = new byte[size];
-            int offset = 0;
-            while (offset < size) {
-              offset += zis.read(content, offset, size - offset);
-            }
-            is = new ByteArrayInputStream(content);
-          } else {
-            log.debug("Size unknown. Reading {}", zipEntry.getName());
-            try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-              while (true) {
-                int byteRead = zis.read();
-                if (byteRead == -1) {
-                  break;
-                }
-                baos.write(byteRead);
-              }
-              is = new ByteArrayInputStream(baos.toByteArray());
-            }
-          }
-          done = true;
-        }
-      }
-    }
-
-    return is;
-  }
-
   /**
    * Check for any needed libraries being present. On Unix none are needed;
    * on windows they must be present
@@ -2525,4 +2500,53 @@ public final class SliderUtils {
     return EnumSet.range(YarnApplicationState.FINISHED,
         YarnApplicationState.KILLED);
   }
+
+  public static final String DAYS = ".days";
+  public static final String HOURS = ".hours";
+  public static final String MINUTES = ".minutes";
+  public static final String SECONDS = ".seconds";
+
+  /**
+   * Get the time range of a set of keys.
+   * @param conf configuration to read properties from
+   * @param basekey base key to which suffix gets applied
+   * @param defDays
+   * @param defHours
+   * @param defMins
+   * @param defSecs
+   * @return the aggregate time range in seconds
+   */
+  public static long getTimeRange(org.apache.slider.api.resource
+      .Configuration conf,
+      String basekey,
+      long defDays,
+      long defHours,
+      long defMins,
+      long defSecs) {
+    Preconditions.checkArgument(basekey != null);
+    long days = conf.getPropertyLong(basekey + DAYS, defDays);
+    long hours = conf.getPropertyLong(basekey + HOURS, defHours);
+
+    long minutes = conf.getPropertyLong(basekey + MINUTES, defMins);
+    long seconds = conf.getPropertyLong(basekey + SECONDS, defSecs);
+    // range check
+    Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
+            && seconds >= 0,
+        "Time range for %s has negative time component %s:%s:%s:%s",
+        basekey, days, hours, minutes, seconds);
+
+    // calculate total time, schedule the reset if expected
+    long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
+    return totalMinutes * 60 + seconds;
+  }
+
+  public static void resolve(Application application) {
+    org.apache.slider.api.resource.Configuration global = application
+        .getConfiguration();
+    for (Component component : application.getComponents()) {
+      mergeMapsIgnoreDuplicateKeys(component.getConfiguration().getProperties(),
+          global.getProperties());
+    }
+    // TODO merge other information to components
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 4182459..965ea35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory;
 
 import java.lang.reflect.Method;
 import java.lang.reflect.InvocationTargetException;
-import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index ef96c9b..4302530 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -337,5 +337,12 @@ public class ZKIntegration implements Watcher, Closeable {
     return SVC_SLIDER_USERS + "/" + username;
   }
 
-
+  /**
+   * Blocking enum of users.
+   * @return an unordered list of clusters under a user
+   */
+  public List<String> getClusters() throws KeeperException,
+      InterruptedException {
+    return zookeeper.getChildren(userPath, null);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index e0299e7..cb39368 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -18,6 +18,7 @@
 
 package org.apache.slider.providers;
 
+import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.resource.Component;
 
 /**
@@ -36,6 +37,14 @@ public final class ProviderRole {
   public final String labelExpression;
   public final Component component;
 
+  public ProviderRole(String name, int id) {
+    this(name,
+        id,
+        PlacementPolicy.DEFAULT,
+        ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
+        ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
+        ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
+  }
 
   /**
    * Create a provider role
@@ -59,7 +68,8 @@ public final class ProviderRole {
         policy,
         nodeFailureThreshold,
         placementTimeoutSeconds,
-        labelExpression, null);
+        labelExpression,
+        new Component().name(name).numberOfContainers(0L));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 4aeffa6..7473dab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -97,7 +97,6 @@ import org.apache.slider.common.tools.PortScanner;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.common.tools.SliderVersionInfo;
-import org.apache.slider.core.conf.MapOperations;
 import org.apache.slider.core.exceptions.BadConfigException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.exceptions.SliderInternalStateException;
@@ -855,7 +854,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     providerService.setAMState(stateForProviders);
 
     // chaos monkey
-//    maybeStartMonkey();
+    maybeStartMonkey();
 
     // if not a secure cluster, extract the username -it will be
     // propagated to workers
@@ -1597,7 +1596,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
    * @throws SliderException slider problems, including invalid configs
    * @throws IOException IO problems
    */
-  public void flexCluster(Messages.FlexComponentRequestProto request)
+  public void flexCluster(Messages.FlexComponentsRequestProto request)
       throws IOException, SliderException {
     if (request != null) {
       appState.updateComponents(request);
@@ -1619,24 +1618,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
 
     ResetFailureWindow reset = new ResetFailureWindow(rmOperationHandler);
 
-    long days =
-        conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".days",
-            ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS);
-    long hours =
-        conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".hours",
-            ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS);
-    long minutes =
-        conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".minutes",
-            ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES);
-    long seconds =
-        conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".seconds",
-            0);
-    Preconditions
-        .checkState(days >= 0 && hours >= 0 && minutes >= 0 && seconds >= 0,
-            "Time range for has negative time component %s:%s:%s:%s", days,
-            hours, minutes, seconds);
-    long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
-    long totalSeconds = totalMinutes * 60 + seconds;
+    long totalSeconds = SliderUtils.getTimeRange(conf,
+        ResourceKeys.CONTAINER_FAILURE_WINDOW,
+        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS,
+        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS,
+        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES,
+        0);
     if (totalSeconds > 0) {
       log.info("Scheduling the failure window reset interval to every {}"
               + " seconds", totalSeconds);
@@ -1810,12 +1797,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
       signalAMComplete(new ActionStopSlider("stop", EXIT_EXCEPTION_THROWN,
           FinalApplicationStatus.FAILED,
-          "AMRMClientAsync.onError() received " + e));
+          SliderUtils.extractFirstLine(e.getLocalizedMessage())));
     } else if (e instanceof InvalidApplicationMasterRequestException) {
       // halt the AM
       LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
       queue(new ActionHalt(EXIT_EXCEPTION_THROWN,
-          "AMRMClientAsync.onError() received " + e));
+          SliderUtils.extractFirstLine(e.getLocalizedMessage())));
     } else {
       // ignore and log
       LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e);
@@ -2040,7 +2027,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
    */
   public void onExceptionInThread(Thread thread, Throwable exception) {
     log.error("Exception in {}: {}", thread.getName(), exception, exception);
-    
+
     // if there is a teardown in progress, ignore it
     if (amCompletionFlag.get()) {
       log.info("Ignoring exception: shutdown in progress");
@@ -2052,26 +2039,27 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       signalAMComplete(new ActionStopSlider("stop",
           exitCode,
           FinalApplicationStatus.FAILED,
-          exception.toString()));
+          SliderUtils.extractFirstLine(exception.getLocalizedMessage())));
     }
   }
 
   /**
-   * TODO Start the chaos monkey
+   * TODO Read chaos monkey params from AM configuration rather than app
+   * configuration
    * @return true if it started
    */
   private boolean maybeStartMonkey() {
-//    MapOperations internals = getGlobalInternalOptions();
-    MapOperations internals = new MapOperations();
-    Boolean enabled =
-        internals.getOptionBool(InternalKeys.CHAOS_MONKEY_ENABLED,
-            InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
+    org.apache.slider.api.resource.Configuration configuration =
+        application.getConfiguration();
+    boolean enabled = configuration.getPropertyBool(
+        InternalKeys.CHAOS_MONKEY_ENABLED,
+        InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
     if (!enabled) {
       log.debug("Chaos monkey disabled");
       return false;
     }
     
-    long monkeyInterval = internals.getTimeRange(
+    long monkeyInterval = SliderUtils.getTimeRange(configuration,
         InternalKeys.CHAOS_MONKEY_INTERVAL,
         InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
         InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
@@ -2083,7 +2071,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       return false;
     }
 
-    long monkeyDelay = internals.getTimeRange(
+    long monkeyDelay = SliderUtils.getTimeRange(configuration,
         InternalKeys.CHAOS_MONKEY_DELAY,
         0,
         0,
@@ -2098,10 +2086,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     // configure the targets
     
     // launch failure: special case with explicit failure triggered now
-    int amLaunchFailProbability = internals.getOptionInt(
+    int amLaunchFailProbability = configuration.getPropertyInt(
         InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
         0);
-    if (amLaunchFailProbability> 0 && monkey.chaosCheck(amLaunchFailProbability)) {
+    if (amLaunchFailProbability > 0 && monkey.chaosCheck(
+        amLaunchFailProbability)) {
       log.info("Chaos Monkey has triggered AM Launch failure");
       // trigger a failure
       ActionStopSlider stop = new ActionStopSlider("stop",
@@ -2112,12 +2101,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       queue(stop);
     }
     
-    int amKillProbability = internals.getOptionInt(
+    int amKillProbability = configuration.getPropertyInt(
         InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
         InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
     monkey.addTarget("AM killer",
         new ChaosKillAM(actionQueues, -1), amKillProbability);
-    int containerKillProbability = internals.getOptionInt(
+    int containerKillProbability = configuration.getPropertyInt(
         InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
         InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
     monkey.addTarget("Container killer",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
index 220f2ca..a7b94ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
@@ -26,9 +26,9 @@ import java.util.concurrent.TimeUnit;
 
 public class ActionFlexCluster extends AsyncAction {
 
-  final Messages.FlexComponentRequestProto requestProto;
+  final Messages.FlexComponentsRequestProto requestProto;
   public ActionFlexCluster(String name, long delay, TimeUnit timeUnit,
-      Messages.FlexComponentRequestProto requestProto) {
+      Messages.FlexComponentsRequestProto requestProto) {
     super(name, delay, timeUnit, ATTR_CHANGES_APP_SIZE);
     this.requestProto = requestProto;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java
index 510ff73..5dcbe9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java
@@ -37,13 +37,13 @@ import static org.apache.hadoop.metrics2.lib.Interns.info;
 @Metrics(context = "yarn-native-service")
 public class SliderMetrics implements MetricsSource {
 
-  @Metric("containers pending")
-  public MutableGaugeInt containersPending;
+  @Metric("containers requested")
+  public MutableGaugeInt containersRequested;
   @Metric("anti-affinity containers pending")
   public MutableGaugeInt pendingAAContainers;
-  @Metric("containers pending")
+  @Metric("containers running")
   public MutableGaugeInt containersRunning;
-  @Metric("containers requested")
+  @Metric("containers desired")
   public MutableGaugeInt containersDesired;
   @Metric("containers completed")
   public MutableGaugeInt containersCompleted;
@@ -53,8 +53,12 @@ public class SliderMetrics implements MetricsSource {
   public MutableGaugeInt failedSinceLastThreshold;
   @Metric("containers preempted")
   public MutableGaugeInt containersPreempted;
+  @Metric("containers exceeded limits")
+  public MutableGaugeInt containersLimitsExceeded;
   @Metric("containers surplus")
   public MutableGaugeInt surplusContainers;
+  @Metric("containers failed due to disk failure")
+  public MutableGaugeInt containersDiskFailure;
 
   protected final MetricsRegistry registry;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
index 7830a1e..526ab7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
@@ -70,11 +70,11 @@ public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB {
   }
 
   @Override
-  public Messages.FlexComponentResponseProto flexComponent(
-      RpcController controller, Messages.FlexComponentRequestProto request)
+  public Messages.FlexComponentsResponseProto flexComponents(
+      RpcController controller, Messages.FlexComponentsRequestProto request)
       throws ServiceException {
     try {
-      return real.flexComponent(request);
+      return real.flexComponents(request);
     } catch (IOException e) {
       throw wrap(e);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
index 1902ec1..2e40a9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
@@ -110,10 +110,10 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol {
   }
 
   @Override
-  public Messages.FlexComponentResponseProto flexComponent(
-      Messages.FlexComponentRequestProto request) throws IOException {
+  public Messages.FlexComponentsResponseProto flexComponents(
+      Messages.FlexComponentsRequestProto request) throws IOException {
     try {
-      return endpoint.flexComponent(NULL_CONTROLLER, request);
+      return endpoint.flexComponents(NULL_CONTROLLER, request);
     } catch (ServiceException e) {
       throw convert(e);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
index eaa0a81..f88d586 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
@@ -196,11 +196,11 @@ public class SliderIPCService extends AbstractService
   }
 
   @Override
-  public Messages.FlexComponentResponseProto flexComponent(
-      Messages.FlexComponentRequestProto request) throws IOException {
+  public Messages.FlexComponentsResponseProto flexComponents(
+      Messages.FlexComponentsRequestProto request) throws IOException {
     onRpcCall("flex");
     schedule(new ActionFlexCluster("flex", 1, TimeUnit.MILLISECONDS, request));
-    return Messages.FlexComponentResponseProto.newBuilder().build();
+    return Messages.FlexComponentsResponseProto.newBuilder().build();
   }
 
   @Override //SliderClusterProtocol

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
index 37c730f..cc19eee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -16,9 +16,23 @@
  */
 package org.apache.slider.server.appmaster.security;
 
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import static org.apache.slider.core.main.LauncherExitCodes.EXIT_UNAUTHORIZED;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.IOException;
+
 /**
  * Class keeping code security information
  */
@@ -26,111 +40,122 @@ public class SecurityConfiguration {
 
   protected static final Logger log =
       LoggerFactory.getLogger(SecurityConfiguration.class);
+  private final Configuration configuration;
+  private final Application application;
   private String clusterName;
 
-//  private void validate() throws SliderException {
-//    if (isSecurityEnabled()) {
-//      String principal = instanceDefinition.getAppConfOperations()
-//          .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
-//      if(SliderUtils.isUnset(principal)) {
-//        // if no login identity is available, fail
-//        UserGroupInformation loginUser = null;
-//        try {
-//          loginUser = getLoginUser();
-//        } catch (IOException e) {
-//          throw new SliderException(EXIT_UNAUTHORIZED, e,
-//                                    "No principal configured for the application and "
-//                                    + "exception raised during retrieval of login user. "
-//                                    + "Unable to proceed with application "
-//                                    + "initialization.  Please ensure a value "
-//                                    + "for %s exists in the application "
-//                                    + "configuration or the login issue is addressed",
-//                                    SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
-//        }
-//        if (loginUser == null) {
-//          throw new SliderException(EXIT_UNAUTHORIZED,
-//                                    "No principal configured for the application "
-//                                    + "and no login user found. "
-//                                    + "Unable to proceed with application "
-//                                    + "initialization.  Please ensure a value "
-//                                    + "for %s exists in the application "
-//                                    + "configuration or the login issue is addressed",
-//                                    SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
-//        }
-//      }
-//      // ensure that either local or distributed keytab mechanism is enabled,
-//      // but not both
-//      String keytabFullPath = instanceDefinition.getAppConfOperations()
-//          .getComponent(SliderKeys.COMPONENT_AM)
-//          .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
-//      String keytabName = instanceDefinition.getAppConfOperations()
-//          .getComponent(SliderKeys.COMPONENT_AM)
-//          .get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
-//      if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
-//        throw new SliderException(EXIT_UNAUTHORIZED,
-//                                  "Both a keytab on the cluster host (%s) and a"
-//                                  + " keytab to be retrieved from HDFS (%s) are"
-//                                  + " specified.  Please configure only one keytab"
-//                                  + " retrieval mechanism.",
-//                                  SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
-//                                  SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
-//
-//      }
-//    }
-//  }
-//
-//  protected UserGroupInformation getLoginUser() throws IOException {
-//    return UserGroupInformation.getLoginUser();
-//  }
-//
-//  public boolean isSecurityEnabled () {
-//    return SliderUtils.isHadoopClusterSecure(configuration);
-//  }
-//
-//  public String getPrincipal () throws IOException {
-//    String principal = instanceDefinition.getAppConfOperations()
-//        .getComponent(SliderKeys.COMPONENT_AM).get(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
-//    if (SliderUtils.isUnset(principal)) {
-//      principal = UserGroupInformation.getLoginUser().getShortUserName();
-//      log.info("No principal set in the slider configuration.  Will use AM login"
-//               + " identity {} to attempt keytab-based login", principal);
-//    }
-//
-//    return principal;
-//  }
-//
-//  public boolean isKeytabProvided() {
-//    boolean keytabProvided = instanceDefinition.getAppConfOperations()
-//                    .getComponent(SliderKeys.COMPONENT_AM)
-//                    .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH) != null ||
-//                instanceDefinition.getAppConfOperations()
-//                    .getComponent(SliderKeys.COMPONENT_AM).
-//                    get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME) != null;
-//    return keytabProvided;
-//
-//  }
-//
-//  public File getKeytabFile(AggregateConf instanceDefinition)
-//      throws SliderException, IOException {
-//    //TODO implement this for dash semantic
-//    String keytabFullPath = instanceDefinition.getAppConfOperations()
-//        .getComponent(SliderKeys.COMPONENT_AM)
-//        .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
-//    File localKeytabFile;
-//    if (SliderUtils.isUnset(keytabFullPath)) {
-//      // get the keytab
-//      String keytabName = instanceDefinition.getAppConfOperations()
-//          .getComponent(SliderKeys.COMPONENT_AM).
-//              get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
-//      log.info("No host keytab file path specified. Will attempt to retrieve"
-//               + " keytab file {} as a local resource for the container",
-//               keytabName);
-//      // download keytab to local, protected directory
-//      localKeytabFile = new File(SliderKeys.KEYTAB_DIR, keytabName);
-//    } else {
-//      log.info("Using host keytab file {} for login", keytabFullPath);
-//      localKeytabFile = new File(keytabFullPath);
-//    }
-//    return localKeytabFile;
-//  }
+  public SecurityConfiguration(Configuration configuration,
+                               Application application,
+                               String clusterName) throws SliderException {
+    Preconditions.checkNotNull(configuration);
+    Preconditions.checkNotNull(application);
+    Preconditions.checkNotNull(clusterName);
+    this.configuration = configuration;
+    this.application = application;
+    this.clusterName = clusterName;
+    validate();
+  }
+
+  private void validate() throws SliderException {
+    if (isSecurityEnabled()) {
+      // TODO use AM configuration rather than app configuration
+      String principal = application.getConfiguration().getProperty(
+          SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+      if(SliderUtils.isUnset(principal)) {
+        // if no login identity is available, fail
+        UserGroupInformation loginUser = null;
+        try {
+          loginUser = getLoginUser();
+        } catch (IOException e) {
+          throw new SliderException(EXIT_UNAUTHORIZED, e,
+              "No principal configured for the application and "
+                  + "exception raised during retrieval of login user. "
+                  + "Unable to proceed with application "
+                  + "initialization.  Please ensure a value "
+                  + "for %s exists in the application "
+                  + "configuration or the login issue is addressed",
+              SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+        }
+        if (loginUser == null) {
+          throw new SliderException(EXIT_UNAUTHORIZED,
+              "No principal configured for the application "
+                  + "and no login user found. "
+                  + "Unable to proceed with application "
+                  + "initialization.  Please ensure a value "
+                  + "for %s exists in the application "
+                  + "configuration or the login issue is addressed",
+              SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+        }
+      }
+      // ensure that either local or distributed keytab mechanism is enabled,
+      // but not both
+      String keytabFullPath = application.getConfiguration().getProperty(
+          SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+      String keytabName = application.getConfiguration().getProperty(
+          SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+      if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
+        throw new SliderException(EXIT_UNAUTHORIZED,
+            "Both a keytab on the cluster host (%s) and a"
+                + " keytab to be retrieved from HDFS (%s) are"
+                + " specified.  Please configure only one keytab"
+                + " retrieval mechanism.",
+            SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH,
+            SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+
+      }
+    }
+  }
+
+  protected UserGroupInformation getLoginUser() throws IOException {
+    return UserGroupInformation.getLoginUser();
+  }
+
+  public boolean isSecurityEnabled() {
+    return SliderUtils.isHadoopClusterSecure(configuration);
+  }
+
+  public String getPrincipal() throws IOException {
+    String principal = application.getConfiguration().getProperty(
+        SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
+    if (SliderUtils.isUnset(principal)) {
+      principal = UserGroupInformation.getLoginUser().getShortUserName();
+      log.info("No principal set in the slider configuration.  Will use AM " +
+          "login identity {} to attempt keytab-based login", principal);
+    }
+
+    return principal;
+  }
+
+  public boolean isKeytabProvided() {
+    String keytabLocalPath = application.getConfiguration().getProperty(
+        SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+    String keytabName = application.getConfiguration().getProperty(
+        SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+    return StringUtils.isNotBlank(keytabLocalPath)
+        || StringUtils.isNotBlank(keytabName);
+
+  }
+
+  public File getKeytabFile()
+      throws SliderException, IOException {
+    //TODO implement this for dash semantic
+    String keytabFullPath = application.getConfiguration().getProperty(
+        SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+    File localKeytabFile;
+    if (SliderUtils.isUnset(keytabFullPath)) {
+      // get the keytab
+      String keytabName = application.getConfiguration().getProperty(
+          SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+      log.info("No host keytab file path specified. Will attempt to retrieve"
+               + " keytab file {} as a local resource for the container",
+               keytabName);
+      // download keytab to local, protected directory
+      localKeytabFile = new File(SliderKeys.KEYTAB_DIR, keytabName);
+    } else {
+      log.info("Using host keytab file {} for login", keytabFullPath);
+      localKeytabFile = new File(keytabFullPath);
+    }
+    return localKeytabFile;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 3d73f3b..43c7ead 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -38,6 +38,7 @@ import org.apache.slider.api.ClusterNode;
 import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.StatusKeys;
 import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.proto.Messages.ComponentCountProto;
 import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.resource.ApplicationState;
 import org.apache.slider.api.resource.Component;
@@ -219,7 +220,13 @@ public class AppState {
     return roleStatusMap;
   }
   
+  protected Map<String, ProviderRole> getRoleMap() {
+    return roles;
+  }
 
+  public Map<Integer, ProviderRole> getRolePriorityMap() {
+    return rolePriorityMap;
+  }
 
   private Map<ContainerId, RoleInstance> getStartingContainers() {
     return startingContainers;
@@ -257,6 +264,11 @@ public class AppState {
     return roleHistory;
   }
 
+  @VisibleForTesting
+  public void setRoleHistory(RoleHistory roleHistory) {
+    this.roleHistory = roleHistory;
+  }
+
   /**
    * Get the path used for history files
    * @return the directory used for history files
@@ -306,6 +318,15 @@ public class AppState {
     appMetrics
         .tag("appId", "Application id for service", app.getId());
 
+    org.apache.slider.api.resource.Configuration conf = app.getConfiguration();
+    startTimeThreshold =
+        conf.getPropertyLong(InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+            InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
+    failureThreshold = conf.getPropertyInt(CONTAINER_FAILURE_THRESHOLD,
+        DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+    nodeFailureThreshold = conf.getPropertyInt(NODE_FAILURE_THRESHOLD,
+        DEFAULT_NODE_FAILURE_THRESHOLD);
+
     //build the initial role list
     List<ProviderRole> roleList = new ArrayList<>(binding.roles);
     for (ProviderRole providerRole : roleList) {
@@ -314,6 +335,7 @@ public class AppState {
 
     int priority = 1;
     for (Component component : app.getComponents()) {
+      priority = getNewPriority(priority);
       String name = component.getName();
       if (roles.containsKey(name)) {
         continue;
@@ -324,22 +346,13 @@ public class AppState {
       }
       log.info("Adding component: " + name);
       ProviderRole dynamicRole =
-          createComponent(name, name, component, priority++);
+          createComponent(name, name, component, priority);
       buildRole(dynamicRole);
       roleList.add(dynamicRole);
     }
     //then pick up the requirements
     buildRoleRequirementsFromResources();
 
-    org.apache.slider.api.resource.Configuration conf = app.getConfiguration();
-    startTimeThreshold =
-        conf.getPropertyLong(InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
-            InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
-    failureThreshold = (int) conf.getPropertyLong(CONTAINER_FAILURE_THRESHOLD,
-        DEFAULT_CONTAINER_FAILURE_THRESHOLD);
-    nodeFailureThreshold = (int) conf.getPropertyLong(NODE_FAILURE_THRESHOLD,
-        DEFAULT_NODE_FAILURE_THRESHOLD);
-
     // set up the role history
     roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory);
     roleHistory.onStart(binding.fs, binding.historyPath);
@@ -359,34 +372,47 @@ public class AppState {
   //TODO WHY do we need to create the component for AM ?
   public ProviderRole createComponent(String name, String group,
       Component component, int priority) throws BadConfigException {
-
     org.apache.slider.api.resource.Configuration conf =
         component.getConfiguration();
     long placementTimeout = conf.getPropertyLong(PLACEMENT_ESCALATE_DELAY,
         DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
     long placementPolicy = conf.getPropertyLong(COMPONENT_PLACEMENT_POLICY,
         PlacementPolicy.DEFAULT);
-    int threshold = (int) conf
-        .getPropertyLong(NODE_FAILURE_THRESHOLD, nodeFailureThreshold);
+    int threshold = conf.getPropertyInt(NODE_FAILURE_THRESHOLD,
+        nodeFailureThreshold);
+    String label = conf.getProperty(YARN_LABEL_EXPRESSION,
+        DEF_YARN_LABEL_EXPRESSION);
     ProviderRole newRole =
         new ProviderRole(name, group, priority, (int)placementPolicy, threshold,
-            placementTimeout, "", component);
+            placementTimeout, label, component);
 
     log.info("Created a new role " + newRole);
     return newRole;
   }
 
-  public synchronized void updateComponents(
-      Messages.FlexComponentRequestProto requestProto)
-      throws BadConfigException {
+  @VisibleForTesting
+  public synchronized List<ProviderRole> updateComponents(Map<String, Long>
+      componentCounts) throws BadConfigException {
     for (Component component : app.getComponents()) {
-      if (component.getName().equals(requestProto.getName())) {
-        component
-            .setNumberOfContainers((long) requestProto.getNumberOfContainers());
+      if (componentCounts.containsKey(component.getName())) {
+        component.setNumberOfContainers(componentCounts.get(component
+            .getName()));
       }
     }
     //TODO update cluster description
-    buildRoleRequirementsFromResources();
+    return buildRoleRequirementsFromResources();
+  }
+
+  public synchronized List<ProviderRole> updateComponents(
+      Messages.FlexComponentsRequestProto requestProto)
+      throws BadConfigException {
+    Map<String, Long> componentCounts = new HashMap<>();
+    for (ComponentCountProto componentCount : requestProto
+        .getComponentsList()) {
+      componentCounts.put(componentCount.getName(), componentCount
+          .getNumberOfContainers());
+    }
+    return updateComponents(componentCounts);
   }
 
   /**
@@ -445,10 +471,8 @@ public class AppState {
     // now the dynamic ones. Iterate through the the cluster spec and
     // add any role status entries not in the role status
 
-    List<RoleStatus> list = new ArrayList<>(getRoleStatusMap().values());
-    for (RoleStatus roleStatus : list) {
-      String name = roleStatus.getName();
-      Component component = roleStatus.getProviderRole().component;
+    for (Component component : app.getComponents()) {
+      String name = component.getName();
       if (roles.containsKey(name)) {
         continue;
       }
@@ -460,10 +484,12 @@ public class AppState {
           groupCount = groupCounts.get(name);
         }
         for (int i = groupCount + 1; i <= desiredInstanceCount; i++) {
-          int priority = roleStatus.getPriority();
           // this is a new instance of an existing group
           String newName = String.format("%s%d", name, i);
-          int newPriority = getNewPriority(priority + i - 1);
+          if (roles.containsKey(newName)) {
+            continue;
+          }
+          int newPriority = getNewPriority(i);
           log.info("Adding new role {}", newName);
           ProviderRole dynamicRole =
               createComponent(newName, name, component, newPriority);
@@ -477,11 +503,12 @@ public class AppState {
         }
       } else {
         // this is a new value
-        log.info("Adding new role {}", name);
+        log.info("Adding new role {}, num containers {}", name,
+            component.getNumberOfContainers());
         ProviderRole dynamicRole =
-            createComponent(name, name, component, roleStatus.getPriority());
+            createComponent(name, name, component, getNewPriority(1));
         RoleStatus newRole = buildRole(dynamicRole);
-        incDesiredContainers(roleStatus,
+        incDesiredContainers(newRole,
             component.getNumberOfContainers().intValue());
         log.info("New role {}", newRole);
         if (roleHistory != null) {
@@ -518,7 +545,8 @@ public class AppState {
     if (roleStatusMap.containsKey(priority)) {
       throw new BadConfigException("Duplicate Provider Key: %s and %s",
                                    providerRole,
-                                   roleStatusMap.get(priority));
+                                   roleStatusMap.get(priority)
+                                       .getProviderRole());
     }
     RoleStatus roleStatus = new RoleStatus(providerRole);
     roleStatusMap.put(priority, roleStatus);
@@ -536,6 +564,8 @@ public class AppState {
   private void buildRoleResourceRequirements() {
     for (RoleStatus role : roleStatusMap.values()) {
       role.setResourceRequirements(buildResourceRequirements(role));
+      log.info("Setting resource requirements for {} to {}", role.getName(),
+          role.getResourceRequirements());
     }
   }
   /**
@@ -827,7 +857,6 @@ public class AppState {
    * @return the container request to submit or null if there is none
    */
   private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) {
-    incPendingContainers(role);
     if (role.isAntiAffinePlacement()) {
       return createAAContainerRequest(role);
     } else {
@@ -857,28 +886,58 @@ public class AppState {
     return request.getIssuedRequest();
   }
 
-  private void incPendingContainers(RoleStatus role) {
-    role.getComponentMetrics().containersPending.incr();
-    appMetrics.containersPending.incr();
+  @VisibleForTesting
+  public void incRequestedContainers(RoleStatus role) {
+    log.info("Incrementing requested containers for {}", role.getName());
+    role.getComponentMetrics().containersRequested.incr();
+    appMetrics.containersRequested.incr();
   }
 
-  private void decPendingContainers(RoleStatus role) {
-    decPendingContainers(role, 1);
+  private void decRequestedContainers(RoleStatus role) {
+    role.getComponentMetrics().containersRequested.decr();
+    appMetrics.containersRequested.decr();
+    log.info("Decrementing requested containers for {} by {} to {}", role
+        .getName(), 1, role.getComponentMetrics().containersRequested.value());
   }
 
-  private void decPendingContainers(RoleStatus role, int n) {
-    role.getComponentMetrics().containersPending.decr(n);;
-    appMetrics.containersPending.decr(n);
+  private int decRequestedContainersToFloor(RoleStatus role, int delta) {
+    int actual = decMetricToFloor(role.getComponentMetrics()
+        .containersRequested, delta);
+    appMetrics.containersRequested.decr(actual);
+    log.info("Decrementing requested containers for {} by {} to {}", role
+            .getName(), actual, role.getComponentMetrics().containersRequested
+        .value());
+    return actual;
   }
 
+  private int decAAPendingToFloor(RoleStatus role, int delta) {
+    int actual = decMetricToFloor(role.getComponentMetrics()
+        .pendingAAContainers, delta);
+    appMetrics.pendingAAContainers.decr(actual);
+    log.info("Decrementing AA pending containers for {} by {} to {}", role
+        .getName(), actual, role.getComponentMetrics().pendingAAContainers
+        .value());
+    return actual;
+  }
 
-  private void incRunningContainers(RoleStatus role) {
-    role.getComponentMetrics().containersRunning.incr();;
+  private int decMetricToFloor(MutableGaugeInt metric, int delta) {
+    int currentValue = metric.value();
+    int decrAmount = delta;
+    if (currentValue - delta < 0) {
+      decrAmount = currentValue;
+    }
+    metric.decr(decrAmount);
+    return decrAmount;
+  }
+
+  @VisibleForTesting
+  public void incRunningContainers(RoleStatus role) {
+    role.getComponentMetrics().containersRunning.incr();
     appMetrics.containersRunning.incr();
   }
 
   private void decRunningContainers(RoleStatus role) {
-    role.getComponentMetrics().containersRunning.decr();;
+    role.getComponentMetrics().containersRunning.decr();
     appMetrics.containersRunning.decr();
   }
 
@@ -902,26 +961,47 @@ public class AppState {
     appMetrics.containersCompleted.incr();
   }
 
-  private void incFailedContainers(RoleStatus role, ContainerOutcome outcome) {
-    role.getComponentMetrics().containersFailed.incr();
-    appMetrics.containersFailed.incr();
+  @VisibleForTesting
+  public void incFailedContainers(RoleStatus role, ContainerOutcome outcome) {
     switch (outcome) {
     case Preempted:
       appMetrics.containersPreempted.incr();
       role.getComponentMetrics().containersPreempted.incr();
       break;
+    case Disk_failure:
+      appMetrics.containersDiskFailure.incr();
+      appMetrics.containersFailed.incr();
+      role.getComponentMetrics().containersDiskFailure.incr();
+      role.getComponentMetrics().containersFailed.incr();
+      break;
     case Failed:
       appMetrics.failedSinceLastThreshold.incr();
+      appMetrics.containersFailed.incr();
+      role.getComponentMetrics().failedSinceLastThreshold.incr();
+      role.getComponentMetrics().containersFailed.incr();
+      break;
+    case Failed_limits_exceeded:
+      appMetrics.containersLimitsExceeded.incr();
+      appMetrics.failedSinceLastThreshold.incr();
+      appMetrics.containersFailed.incr();
+      role.getComponentMetrics().containersLimitsExceeded.incr();
+      role.getComponentMetrics().failedSinceLastThreshold.incr();
+      role.getComponentMetrics().containersFailed.incr();
       break;
     default:
+      appMetrics.failedSinceLastThreshold.incr();
+      appMetrics.containersFailed.incr();
+      role.getComponentMetrics().failedSinceLastThreshold.incr();
+      role.getComponentMetrics().containersFailed.incr();
       break;
     }
   }
 
   /**
-   * Build up the resource requirements for this role from the
-   * cluster specification, including substituing max allowed values
-   * if the specification asked for it.
+   * Build up the resource requirements for this role from the cluster
+   * specification, including substituting max allowed values if the
+   * specification asked for it (except when
+   * {@link ResourceKeys#YARN_RESOURCE_NORMALIZATION_ENABLED} is set to false).
    * @param role role
    * during normalization
    */
@@ -934,17 +1014,36 @@ public class AppState {
       // TODO why do we need to create the component for AM ?
       return Resource.newInstance(1, 512);
     }
-    int cores = Math.min(containerMaxCores, component.getResource().getCpus());
+    int cores = DEF_YARN_CORES;
+    if (component.getResource() != null && component.getResource().getCpus()
+        != null) {
+      cores = Math.min(containerMaxCores, component.getResource().getCpus());
+    }
     if (cores <= 0) {
       cores = DEF_YARN_CORES;
     }
-    long mem = Math.min(containerMaxMemory,
-        Long.parseLong(component.getResource().getMemory()));
+    long rawMem = DEF_YARN_MEMORY;
+    if (component.getResource() != null && component.getResource().getMemory()
+        != null) {
+      if (YARN_RESOURCE_MAX.equals(component.getResource().getMemory())) {
+        rawMem = containerMaxMemory;
+      } else {
+        rawMem = Long.parseLong(component.getResource().getMemory());
+      }
+    }
+    boolean normalize = component.getConfiguration().getPropertyBool(
+        YARN_RESOURCE_NORMALIZATION_ENABLED, true);
+    if (!normalize) {
+      log.info("Resource normalization: disabled");
+      log.debug("Component {} has RAM={}, vCores={}", name, rawMem, cores);
+      return Resources.createResource(rawMem, cores);
+    }
+    long mem = Math.min(containerMaxMemory, rawMem);
     if (mem <= 0) {
       mem = DEF_YARN_MEMORY;
     }
     Resource capability = Resource.newInstance(mem, cores);
-    log.debug("Component {} has RAM={}, vCores ={}", name, mem, cores);
+    log.debug("Component {} has RAM={}, vCores={}", name, mem, cores);
     Resource normalized = recordFactory.normalize(capability, minResource,
         maxResource);
     if (!Resources.equals(normalized, capability)) {
@@ -1060,7 +1159,7 @@ public class AppState {
       log.debug("Created {} cancel requests", operations.size());
       return new NodeUpdatedOutcome(true, operations);
     }
-    return new NodeUpdatedOutcome(false, new ArrayList<AbstractRMOperation>(0));
+    return new NodeUpdatedOutcome(false, new ArrayList<>(0));
   }
 
   /**
@@ -1203,7 +1302,6 @@ public class AppState {
             message = String.format("Failure %s (%d)", containerId, exitStatus);
           }
           roleStatus.noteFailed(message);
-          incFailedContainers(roleStatus, result.outcome);
           long failed =
               roleStatus.getComponentMetrics().containersFailed.value();
           log.info("Current count of failed role[{}] {} =  {}",
@@ -1409,7 +1507,7 @@ public class AppState {
           role.getName(), failures, threshold);
     }
 
-    if (failures > threshold) {
+    if (threshold > 0 && failures > threshold) {
       throw new TriggerClusterTeardownException(
           SliderExitCodes.EXIT_DEPLOYMENT_FAILED, FinalApplicationStatus.FAILED,
           ErrorStrings.E_UNSTABLE_CLUSTER
@@ -1428,7 +1526,7 @@ public class AppState {
   private int getFailureThresholdForRole(RoleStatus roleStatus) {
     return (int) roleStatus.getProviderRole().component.getConfiguration()
         .getPropertyLong(CONTAINER_FAILURE_THRESHOLD,
-            DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+            failureThreshold);
   }
 
 
@@ -1497,7 +1595,8 @@ public class AppState {
     }
 
     log.info("Reviewing {} : ", role);
-    log.debug("Expected {}, Delta: {}", expected, delta);
+    log.debug("Expected {}, Requested/Running {}, Delta: {}", expected,
+        role.getActualAndRequested(), delta);
     checkFailureThreshold(role);
 
     if (expected < 0 ) {
@@ -1526,7 +1625,7 @@ public class AppState {
               pending--;
               log.info("Starting an anti-affine request sequence for {} nodes; pending={}",
                 delta, pending);
-              addContainerRequest(operations, request);
+              addContainerRequest(operations, request, role);
             } else {
               log.info("No location for anti-affine request");
             }
@@ -1536,12 +1635,12 @@ public class AppState {
         }
         log.info("Setting pending to {}", pending);
         //TODO
-        role.setAAPending((int)pending);
+        role.setAAPending(pending);
       } else {
 
         for (int i = 0; i < delta; i++) {
           //get the role history to select a suitable node, if available
-          addContainerRequest(operations, createContainerRequest(role));
+          addContainerRequest(operations, createContainerRequest(role), role);
         }
       }
     } else if (delta < 0) {
@@ -1552,25 +1651,35 @@ public class AppState {
       long excess = -delta;
 
       // how many requests are outstanding? for AA roles, this includes pending
-      long outstandingRequests = role.getPending() + role.getAAPending();
+      long outstandingRequests = role.getRequested() + role.getAAPending();
       if (outstandingRequests > 0) {
         // outstanding requests.
         int toCancel = (int)Math.min(outstandingRequests, excess);
 
+        int pendingCancelled = 0;
+        if (role.getAAPending() > 0) {
+          pendingCancelled = decAAPendingToFloor(role, toCancel);
+        }
+        int remainingToCancel = toCancel - pendingCancelled;
+
         // Delegate to Role History
-        List<AbstractRMOperation> cancellations = roleHistory.cancelRequestsForRole(role, toCancel);
+        List<AbstractRMOperation> cancellations = roleHistory
+            .cancelRequestsForRole(role, remainingToCancel);
         log.info("Found {} outstanding requests to cancel", cancellations.size());
         operations.addAll(cancellations);
-        if (toCancel != cancellations.size()) {
+        if (remainingToCancel != cancellations.size()) {
           log.error("Tracking of outstanding requests is not in sync with the summary statistics:" +
               " expected to be able to cancel {} requests, but got {}",
-              toCancel, cancellations.size());
+              remainingToCancel, cancellations.size());
         }
-        decPendingContainers(role, toCancel);
-        excess -= toCancel;
+
+        int requestCancelled = decRequestedContainersToFloor(role,
+            remainingToCancel);
+        excess -= pendingCancelled;
+        excess -= requestCancelled;
         assert excess >= 0 : "Attempted to cancel too many requests";
         log.info("Submitted {} cancellations, leaving {} to release",
-            toCancel, excess);
+            pendingCancelled + requestCancelled, excess);
         if (excess == 0) {
           log.info("After cancelling requests, application is now at desired size");
         }
@@ -1645,7 +1754,7 @@ public class AppState {
    * @return true if a request was added
    */
   private boolean addContainerRequest(List<AbstractRMOperation> operations,
-      AMRMClient.ContainerRequest containerAsk) {
+      AMRMClient.ContainerRequest containerAsk, RoleStatus role) {
     if (containerAsk != null) {
       log.info("Container ask is {} and label = {}", containerAsk,
           containerAsk.getNodeLabelExpression());
@@ -1654,6 +1763,7 @@ public class AppState {
         log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
       }
       operations.add(new ContainerRequestOperation(containerAsk));
+      incRequestedContainers(role);
       return true;
     } else {
       return false;
@@ -1727,6 +1837,8 @@ public class AppState {
       List<Container> allocatedContainers,
       List<ContainerAssignment> assignments,
       List<AbstractRMOperation> operations) {
+    assignments.clear();
+    operations.clear();
     List<Container> ordered = roleHistory.prepareAllocationList(allocatedContainers);
     log.info("onContainersAllocated(): Total containers allocated = {}", ordered.size());
     for (Container container : ordered) {
@@ -1735,13 +1847,13 @@ public class AppState {
       //get the role
       final ContainerId cid = container.getId();
       final RoleStatus role = lookupRoleStatus(container);
-      decPendingContainers(role);
+      decRequestedContainers(role);
 
       //inc allocated count -this may need to be dropped in a moment,
       // but us needed to update the logic below
       MutableGaugeInt containersRunning = role.getComponentMetrics().containersRunning;
-      final long allocated = containersRunning.value();
       incRunningContainers(role);
+      final long allocated = containersRunning.value();
       final long desired = role.getDesired();
 
       final String roleName = role.getName();
@@ -1778,7 +1890,8 @@ public class AppState {
           if (role.getAAPending() > 0) {
             // still an outstanding AA request: need to issue a new one.
             log.info("Asking for next container for AA role {}", roleName);
-            if (!addContainerRequest(operations, createAAContainerRequest(role))) {
+            if (!addContainerRequest(operations, createAAContainerRequest(role),
+                role)) {
               log.info("No capacity in cluster for new requests");
             } else {
               role.decAAPending();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message