hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [33/50] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)
Date Tue, 11 Oct 2016 20:37:37 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
new file mode 100644
index 0000000..5140059
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ComponentArgsDelegate extends AbstractArgsDelegate {
+
+  /**
+   * This is a listing of the roles to create
+   */
+  @Parameter(names = {ARG_COMPONENT,  ARG_COMPONENT_SHORT, ARG_ROLE},
+             arity = 2,
+             description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
+             splitter = DontSplitArguments.class)
+  public List<String> componentTuples = new ArrayList<>(0);
+
+
+  /**
+   * Get the role mapping (may be empty, but never null)
+   * @return role mapping
+   * @throws BadCommandArgumentsException parse problem
+   */
+  public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
+    return convertTupleListToMap("component", componentTuples);
+  }
+
+  public List<String> getComponentTuples() {
+    return componentTuples;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
new file mode 100644
index 0000000..0344305
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+  @Override
+  public List<String> split(String value) {
+    List<String> list = new ArrayList<>(1);
+    list.add(value);
+    return list;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
new file mode 100644
index 0000000..7524053
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import java.io.File;
+
+/**
+ * Launch args for create and start and anything else that can start something
+ */
+public interface LaunchArgsAccessor extends WaitTimeAccessor {
+  String getRmAddress();
+
+  File getOutputFile();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
new file mode 100644
index 0000000..bc7e94c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+import java.io.File;
+
+/**
+ * Any launch-time args
+ */
+public class LaunchArgsDelegate extends WaitArgsDelegate implements
+                                                         LaunchArgsAccessor {
+
+
+  //TODO: do we need this?
+  @Parameter(names = ARG_RESOURCE_MANAGER,
+             description = "Resource manager hostname:port ",
+             required = false)
+  private String rmAddress;
+
+  @Override
+  public String getRmAddress() {
+    return rmAddress;
+  }
+
+  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+      description = "output file for any application report")
+  public File outputFile;
+
+  @Override
+  public File getOutputFile() {
+    return outputFile;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
new file mode 100644
index 0000000..ccb526c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+import org.apache.hadoop.fs.Path;
+
+public class PathArgumentConverter extends BaseConverter<Path> {
+
+  public PathArgumentConverter(String optionName) {
+    super(optionName);
+  }
+
+  @Override
+  public Path convert(String value) {
+    return new Path(value);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
new file mode 100644
index 0000000..f9516d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class SliderAMArgs extends CommonArgs {
+
+  SliderAMCreateAction createAction = new SliderAMCreateAction();
+
+  public SliderAMArgs(String[] args) {
+    super(args);
+  }
+
+  @Override
+  protected void addActionArguments() {
+    addActions(createAction);
+  }
+
+  public String getImage() {
+    return createAction.image;
+  }
+
+  /**
+   * This is the URI in the FS to the Slider cluster; the conf file (and any
+   * other cluster-specifics) can be picked up here
+   */
+  public String getSliderClusterURI() {
+    return createAction.sliderClusterURI;
+  }
+
+  /**
+   * Am binding is simple: there is only one action
+   */
+  @Override
+  public void applyAction() {
+    bindCoreAction(createAction);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
new file mode 100644
index 0000000..197c22b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+
+@Parameters(commandNames = {SliderActions.ACTION_CREATE},
+            commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class SliderAMCreateAction extends AbstractActionArgs implements
+                                                           LaunchArgsAccessor {
+
+
+  @Override
+  public String getActionName() {
+    return SliderActions.ACTION_CREATE;
+  }
+
+  @Parameter(names = ARG_IMAGE, description = "image", required = false)
+  public String image;
+
+  /**
+   * This is the URI in the FS to the Slider cluster; the conf file (and any
+   * other cluster-specifics) can be picked up here
+   */
+  @Parameter(names = ARG_CLUSTER_URI,
+             description = "URI to the Slider cluster", required = true)
+  public String sliderClusterURI;
+
+  @ParametersDelegate
+  LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+  @Override
+  public String getRmAddress() {
+    return launchArgs.getRmAddress();
+  }
+
+  @Override
+  public int getWaittime() {
+    return launchArgs.getWaittime();
+  }
+
+  @Override
+  public void setWaittime(int waittime) {
+    launchArgs.setWaittime(waittime);
+  }
+
+  @Override
+  public File getOutputFile() {
+    return launchArgs.getOutputFile();
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
new file mode 100644
index 0000000..204ad9a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+/**
+ * Actions.
+ * Only some of these are supported by specific Slider Services; they
+ * are listed here to ensure the names are consistent
+ */
+public interface SliderActions {
+  String ACTION_AM_SUICIDE = "am-suicide";
+  String ACTION_BUILD = "build";
+  String ACTION_CLIENT = "client";
+  String ACTION_CREATE = "create";
+  String ACTION_DIAGNOSTICS = "diagnostics";
+  String ACTION_DEPENDENCY = "dependency";
+  String ACTION_UPDATE = "update";
+  String ACTION_UPGRADE = "upgrade";
+  String ACTION_DESTROY = "destroy";
+  String ACTION_ECHO = "echo";
+  String ACTION_EXISTS = "exists";
+  String ACTION_FLEX = "flex";
+  String ACTION_FREEZE = "stop";
+  String ACTION_HELP = "help";
+  String ACTION_INSTALL_KEYTAB = "install-keytab";
+  String ACTION_INSTALL_PACKAGE = "install-package";
+  String ACTION_KDIAG = "kdiag";
+  String ACTION_KEYTAB = "keytab";
+  String ACTION_KILL_CONTAINER = "kill-container";
+  String ACTION_LIST = "list";
+  String ACTION_LOOKUP = "lookup";
+  String ACTION_NODES = "nodes";
+  String ACTION_PACKAGE = "package";
+  String ACTION_PREFLIGHT = "preflight";
+  String ACTION_RECONFIGURE = "reconfigure";
+  String ACTION_REGISTRY = "registry";
+  String ACTION_RESOLVE = "resolve";
+  String ACTION_RESOURCE = "resource";
+  String ACTION_STATUS = "status";
+  String ACTION_THAW = "start";
+  String ACTION_TOKENS = "tokens";
+
+  String ACTION_VERSION = "version";
+  String DESCRIBE_ACTION_AM_SUICIDE =
+      "Tell the Slider Application Master to simulate a process failure by terminating itself";
+  String DESCRIBE_ACTION_BUILD =
+    "Build a Slider cluster specification, but do not start it";
+  String DESCRIBE_ACTION_CREATE =
+      "Create a live Slider application";
+  String DESCRIBE_ACTION_DEPENDENCY =
+      "Slider AM and agent dependency (libraries) management";
+  String DESCRIBE_ACTION_UPDATE =
+      "Update template for a Slider application";
+  String DESCRIBE_ACTION_UPGRADE =
+      "Rolling upgrade/downgrade the application to a newer/previous version";
+  String DESCRIBE_ACTION_DESTROY =
+        "Destroy a stopped Slider application";
+  String DESCRIBE_ACTION_EXISTS =
+            "Probe for an application running";
+  String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
+  String DESCRIBE_ACTION_FREEZE =
+              "Stop a running application";
+  String DESCRIBE_ACTION_GETCONF =
+                "Get the configuration of an application";
+  String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
+  String DESCRIBE_ACTION_KILL_CONTAINER =
+    "Kill a container in the application";
+  String DESCRIBE_ACTION_HELP = "Print help information";
+  String DESCRIBE_ACTION_LIST =
+                  "List running Slider applications";
+  String DESCRIBE_ACTION_LOOKUP =
+                  "look up a YARN application";
+  String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
+  String DESCRIBE_ACTION_MONITOR =
+                    "Monitor a running application";
+  String DESCRIBE_ACTION_REGISTRY =
+                      "Query the registry of a YARN application";
+  String DESCRIBE_ACTION_RESOLVE =
+                      "Resolve or list records in the YARN registry";
+  String DESCRIBE_ACTION_STATUS =
+                      "Get the status of an application";
+  String DESCRIBE_ACTION_THAW =
+                        "Start a stopped application";
+  String DESCRIBE_ACTION_VERSION =
+                        "Print the Slider version information";
+  String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." +
+  		" Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.";
+  String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages";
+  String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
+  String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." +
+  		" Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'.";
+  String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
+  String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
+  String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
+
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
new file mode 100644
index 0000000..b0d1ebf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class URIArgumentConverter extends BaseConverter<URI> {
+
+  public URIArgumentConverter(String optionName) {
+    super(optionName);
+  }
+
+  @Override
+  public URI convert(String value) {
+    try {
+      return new URI(value);
+    } catch (URISyntaxException e) {
+      throw new RuntimeException("Cannot make a URI from " + value);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
new file mode 100644
index 0000000..8894309
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class URLArgumentConverter extends BaseConverter<URL> {
+  public URLArgumentConverter(String optionName) {
+    super(optionName);
+  }
+
+  @Override
+  public URL convert(String value) {
+    try {
+      return new URL(value);
+    } catch (MalformedURLException e) {
+      throw new RuntimeException("Cannot make a URL from " + value);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
new file mode 100644
index 0000000..1c27c01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+public class WaitArgsDelegate extends AbstractArgsDelegate implements
+                                                           WaitTimeAccessor {
+
+
+  //--wait [timeout]
+  @Parameter(names = {ARG_WAIT},
+             description = "time to wait for an action to complete")
+  public int waittime = 0;
+
+
+  @Override
+  public int getWaittime() {
+    return waittime;
+  }
+
+  @Override
+  public void setWaittime(int waittime) {
+    this.waittime = waittime;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
new file mode 100644
index 0000000..13d4d5a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+public interface WaitTimeAccessor {
+  int getWaittime();
+  void setWaittime(int waittime);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
new file mode 100644
index 0000000..6380d0c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Some general comparators
+ */
+public class Comparators {
+
+  public static class LongComparator implements Comparator<Long>, Serializable {
+    @Override
+    public int compare(Long o1, Long o2) {
+      long result = o1 - o2;
+      // need to comparisons with a diff greater than integer size
+      if (result < 0 ) {
+        return -1;
+      } else if (result > 0) {
+        return 1;
+      }
+      return 0;
+    }
+  }
+public static class InvertedLongComparator implements Comparator<Long>, Serializable {
+  private static final LongComparator inner = new LongComparator();
+    @Override
+    public int compare(Long o1, Long o2) {
+      return -inner.compare(o1, o2);
+    }
+  }
+
+
+  /**
+   * Little template class to reverse any comparitor
+   * @param <CompareType> the type that is being compared
+   */
+  public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+      Serializable {
+
+    final Comparator<CompareType> instance;
+
+    public ComparatorReverser(Comparator<CompareType> instance) {
+      this.instance = instance;
+    }
+
+    @Override
+    public int compare(CompareType first, CompareType second) {
+      return instance.compare(second, first);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
new file mode 100644
index 0000000..9db241d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -0,0 +1,658 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Methods to aid in config, both in the Configuration class and
+ * with other parts of setting up Slider-initated processes.
+ * 
+ * Some of the methods take an argument of a map iterable for their sources; this allows
+ * the same method
+ */
+public class ConfigHelper {
+  private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
+
+  private static AtomicBoolean sliderResourceInjected =
+      new AtomicBoolean(false);
+  private static AtomicBoolean sliderResourceInjectionAttempted =
+      new AtomicBoolean(false);
+  
+  /**
+   * Dump the (sorted) configuration
+   * @param conf config
+   * @return the sorted keyset
+   */
+  public static Set<String> dumpConf(Configuration conf) {
+    Set<String> keys = sortedConfigKeys(conf);
+    for (String key : keys) {
+      log.info("{}={}", key, conf.get(key));
+    }
+    return keys;
+  }
+
+  /**
+   * Take a configuration and return a sorted set
+   * @param conf config
+   * @return the sorted keyset
+
+   */
+  public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
+    TreeSet<String> sorted = new TreeSet<String>();
+    for (Map.Entry<String, String> entry : conf) {
+      sorted.add(entry.getKey());
+    }
+    return sorted;
+  }
+
+  /**
+   * Set an entire map full of values
+   *
+   * @param config config to patch
+   * @param map map of data
+   * @param origin origin data
+   */
+  public static void addConfigMap(Configuration config,
+                                  Map<String, String> map,
+                                  String origin) throws BadConfigException {
+    addConfigMap(config, map.entrySet(), origin);
+  }
+  
+  /**
+   * Set an entire map full of values
+   *
+   * @param config config to patch
+   * @param map map of data
+   * @param origin origin data
+   */
+  public static void addConfigMap(Configuration config,
+                                  Iterable<Map.Entry<String, String>> map,
+                                  String origin) throws BadConfigException {
+    for (Map.Entry<String, String> mapEntry : map) {
+      String key = mapEntry.getKey();
+      String value = mapEntry.getValue();
+      if (value == null) {
+        throw new BadConfigException("Null value for property " + key);
+      }
+      config.set(key, value, origin);
+    }
+  }
+
+
+  /**
+   * Save a config file in a destination directory on a given filesystem
+   * @param systemConf system conf used for creating filesystems
+   * @param confToSave config to save
+   * @param confdir the directory path where the file is to go
+   * @param filename the filename
+   * @return the destination path where the file was saved
+   * @throws IOException IO problems
+   */
+  public static Path saveConfig(Configuration systemConf,
+                                Configuration confToSave,
+                                Path confdir,
+                                String filename) throws IOException {
+    FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+    Path destPath = new Path(confdir, filename);
+    saveConfig(fs, destPath, confToSave);
+    return destPath;
+  }
+
+  /**
+   * Save a config
+   * @param fs filesystem
+   * @param destPath dest to save
+   * @param confToSave  config to save
+   * @throws IOException IO problems
+   */
+  public static void saveConfig(FileSystem fs,
+                                Path destPath,
+                                Configuration confToSave) throws
+                                                              IOException {
+    FSDataOutputStream fos = fs.create(destPath);
+    try {
+      confToSave.writeXml(fos);
+    } finally {
+      IOUtils.closeStream(fos);
+    }
+  }
+
+  /**
+   * Convert to an XML string
+   * @param conf configuration
+   * @return conf
+   * @throws IOException
+   */
+  public static String toXml(Configuration conf) throws IOException {
+    StringWriter writer = new StringWriter();
+    conf.writeXml(writer);
+    return writer.toString();
+  }
+  
+  /**
+   * This will load and parse a configuration to an XML document
+   * @param fs filesystem
+   * @param path path
+   * @return an XML document
+   * @throws IOException IO failure
+   */
+  public Document parseConfiguration(FileSystem fs,
+                                     Path path) throws
+                                                IOException {
+
+
+    byte[] data = loadBytes(fs, path);
+    //this is here to track down a parse issue
+    //related to configurations
+    String s = new String(data, 0, data.length);
+    log.debug("XML resource {} is \"{}\"", path, s);
+/* JDK7
+    try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+      Document document = parseConfigXML(in);
+      return document;
+    } catch (ParserConfigurationException | SAXException e) {
+      throw new IOException(e);
+    }
+*/
+    ByteArrayInputStream in= null;
+    try {
+      in = new ByteArrayInputStream(data);
+      Document document = parseConfigXML(in);
+      return document;
+    } catch (ParserConfigurationException e) {
+      throw new IOException(e);
+    } catch (SAXException e) {
+      throw new IOException(e);
+    } finally {
+      IOUtils.closeStream(in);
+    }
+  }
+
+  public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
+    int len = (int) fs.getLength(path);
+    byte[] data = new byte[len];
+    /* JDK7
+    try(FSDataInputStream in = fs.open(path)) {
+      in.readFully(0, data);
+    }
+*/
+    FSDataInputStream in = null;
+    in = fs.open(path);
+    try {
+      in.readFully(0, data);
+    } finally {
+      IOUtils.closeStream(in);
+    }
+    return data;
+  }
+
+  /**
+   * Load a configuration from ANY FS path. The normal Configuration
+   * loader only works with file:// URIs
+   * @param fs filesystem
+   * @param path path
+   * @return a loaded resource
+   * @throws IOException
+   */
+  public static Configuration loadConfiguration(FileSystem fs,
+                                                Path path) throws IOException {
+    byte[] data = loadBytes(fs, path);
+
+    ByteArrayInputStream in2;
+
+    in2 = new ByteArrayInputStream(data);
+    Configuration conf1 = new Configuration(false);
+    conf1.addResource(in2);
+    //now clone it while dropping all its sources
+    Configuration conf2   = new Configuration(false);
+    String src = path.toString();
+    for (Map.Entry<String, String> entry : conf1) {
+      String key = entry.getKey();
+      String value = entry.getValue();
+      conf2.set(key, value, src);
+    }
+    return conf2;
+  }
+
+
+  /**
+   * Generate a config file in a destination directory on the local filesystem
+   * @param confdir the directory path where the file is to go
+   * @param filename the filename
+   * @return the destination path
+   */
+  public static File saveConfig(Configuration generatingConf,
+                                    File confdir,
+                                    String filename) throws IOException {
+
+
+    File destPath = new File(confdir, filename);
+    OutputStream fos = new FileOutputStream(destPath);
+    try {
+      generatingConf.writeXml(fos);
+    } finally {
+      IOUtils.closeStream(fos);
+    }
+    return destPath;
+  }
+
+  /**
+   * Parse an XML Hadoop configuration into an XML document. x-include
+   * is supported, but as the location isn't passed in, relative
+   * URIs are out.
+   * @param in instream
+   * @return a document
+   * @throws ParserConfigurationException parser feature problems
+   * @throws IOException IO problems
+   * @throws SAXException XML is invalid
+   */
+  public static Document parseConfigXML(InputStream in) throws
+                                               ParserConfigurationException,
+                                               IOException,
+                                               SAXException {
+    DocumentBuilderFactory docBuilderFactory
+      = DocumentBuilderFactory.newInstance();
+    //ignore all comments inside the xml file
+    docBuilderFactory.setIgnoringComments(true);
+
+    //allow includes in the xml file
+    docBuilderFactory.setNamespaceAware(true);
+    docBuilderFactory.setXIncludeAware(true);
+    DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
+    return builder.parse(in);
+  }
+
+  /**
+   * Load a Hadoop configuration from a local file.
+   * @param file file to load
+   * @return a configuration which hasn't actually had the load triggered
+   * yet.
+   * @throws FileNotFoundException file is not there
+   * @throws IOException any other IO problem
+   */
+  public static Configuration loadConfFromFile(File file) throws
+                                                          IOException {
+    return loadConfFromFile(file, false);
+  }
+
+  /**
+   *
+   * Load a Hadoop configuration from a local file.
+   * @param file file to load
+   * @param loadDefaults flag to indicate if the defaults should be loaded yet
+   * @return a configuration which hasn't actually had the load triggered
+   * yet.
+   * @throws FileNotFoundException file is not there
+   * @throws IOException any other IO problem
+   */
+  public static Configuration loadConfFromFile(File file,
+      boolean loadDefaults) throws IOException {
+    if (!file.exists()) {
+      throw new FileNotFoundException("File not found :"
+                                          + file.getAbsoluteFile());
+    }
+    Configuration conf = new Configuration(loadDefaults);
+    try {
+      conf.addResource(file.toURI().toURL());
+    } catch (MalformedURLException e) {
+      // should never happen...
+      throw new IOException(
+        "File " + file.toURI() + " doesn't have a valid URL");
+    }
+    return conf;
+  }
+
+  /**
+   * Add a configuration from a file to an existing configuration
+   * @param conf existing configuration
+   * @param file file to load
+   * @param overwrite flag to indicate new values should overwrite the predecessor
+   * @return the merged configuration
+   * @throws IOException
+   */
+  public static Configuration addConfigurationFile(Configuration conf,
+      File file, boolean overwrite)
+      throws IOException {
+    Configuration c2 = loadConfFromFile(file, false);
+    mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
+    return conf;
+  }
+
+  /**
+   * Add the system env variables with the given prefix (by convention, env.)
+   * @param conf existing configuration
+   * @param prefix prefix
+   */
+  public static void addEnvironmentVariables(Configuration conf, String prefix) {
+    Map<String, String> env = System.getenv();
+    for (Map.Entry<String, String> entry : env.entrySet()) {
+      conf.set(prefix + entry.getKey(),entry.getValue(), "env");
+    }
+  }
+  
+  /**
+   * looks for the config under $confdir/$templateFilename; if not there
+   * loads it from /conf/templateFile.
+   * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+   * origin to help debug what's happening
+   * @param systemConf system conf
+   * @param confdir conf dir in FS
+   * @param templateFilename filename in the confdir
+   * @param fallbackResource resource to fall back on
+   * @return loaded conf
+   * @throws IOException IO problems
+   */
+  public static Configuration loadTemplateConfiguration(Configuration systemConf,
+                                                        Path confdir,
+                                                        String templateFilename,
+                                                        String fallbackResource) throws
+                                                                         IOException {
+    FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+
+    Path templatePath = new Path(confdir, templateFilename);
+    return loadTemplateConfiguration(fs, templatePath, fallbackResource);
+  }
+
+  /**
+   * looks for the config under $confdir/$templateFilename; if not there
+   * loads it from /conf/templateFile.
+   * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+   * origin to help debug what's happening.
+   * @param fs Filesystem
+   * @param templatePath HDFS path for template
+   * @param fallbackResource resource to fall back on, or "" for no fallback
+   * @return loaded conf
+   * @throws IOException IO problems
+   * @throws FileNotFoundException if the path doesn't have a file and there
+   * was no fallback.
+   */
+  public static Configuration loadTemplateConfiguration(FileSystem fs,
+                                                        Path templatePath,
+                                                        String fallbackResource)
+      throws IOException {
+    Configuration conf;
+    String origin;
+    if (fs.exists(templatePath)) {
+      log.debug("Loading template configuration {}", templatePath);
+      conf = loadConfiguration(fs, templatePath);
+      origin = templatePath.toString();
+    } else {
+      if (fallbackResource.isEmpty()) {
+        throw new FileNotFoundException("No config file found at " + templatePath);
+      }
+      log.debug("Template {} not found" +
+                " -reverting to classpath resource {}", templatePath, fallbackResource);
+      conf = new Configuration(false);
+      conf.addResource(fallbackResource);
+      origin = "Resource " + fallbackResource;
+    }
+    //force a get
+    conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
+    //now set the origin
+    conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
+    return conf;
+  }
+
+
+  /**
+   * For testing: dump a configuration
+   * @param conf configuration
+   * @return listing in key=value style
+   */
+  public static String dumpConfigToString(Configuration conf) {
+    Set<String> sorted = sortedConfigKeys(conf);
+
+    StringBuilder builder = new StringBuilder();
+    for (String key : sorted) {
+
+      builder.append(key)
+             .append("=")
+             .append(conf.get(key))
+             .append("\n");
+    }
+    return builder.toString();
+  }
+
+  /**
+   * Merge in one configuration above another
+   * @param base base config
+   * @param merge one to merge. This MUST be a non-default-load config to avoid
+   * merge origin confusion
+   * @param origin description of the origin for the put operation
+   * @param overwrite flag to indicate new values should overwrite the predecessor
+   * @return the base with the merged values
+   */
+  public static Configuration mergeConfigurations(Configuration base,
+      Iterable<Map.Entry<String, String>> merge,
+      String origin,
+      boolean overwrite) {
+    for (Map.Entry<String, String> entry : merge) {
+      String key = entry.getKey();
+      if (overwrite || base.get(key) == null) {
+        base.set(key, entry.getValue(), origin);
+      }
+    }
+    return base;
+  }
+
+  /**
+   * Register a resource as a default resource.
+   * Do not attempt to use this unless you understand that the
+   * order in which default resources are loaded affects the outcome,
+   * and that subclasses of Configuration often register new default
+   * resources
+   * @param resource the resource name
+   * @return the URL or null
+   */
+  public static URL registerDefaultResource(String resource) {
+    URL resURL = getResourceUrl(resource);
+    if (resURL != null) {
+      Configuration.addDefaultResource(resource);
+    }
+    return resURL;
+  }
+
+  /**
+   * Load a configuration from a resource on this classpath.
+   * If the resource is not found, an empty configuration is returned
+   * @param resource the resource name
+   * @return the loaded configuration.
+   */
+  public static Configuration loadFromResource(String resource) {
+    Configuration conf = new Configuration(false);
+    URL resURL = getResourceUrl(resource);
+    if (resURL != null) {
+      log.debug("loaded resources from {}", resURL);
+      conf.addResource(resource);
+    } else{
+      log.debug("failed to find {} on the classpath", resource);
+    }
+    return conf;
+    
+  }
+
+  /**
+   * Get the URL to a resource, null if not on the CP
+   * @param resource resource to look for
+   * @return the URL or null
+   */
+  public static URL getResourceUrl(String resource) {
+    return ConfigHelper.class.getClassLoader()
+                                  .getResource(resource);
+  }
+
+  /**
+   * Load a resource that must be on the classpath
+   * @param resource the resource name
+   * @return the loaded configuration
+   * @throws FileNotFoundException if the resource is missing
+   */
+  public static Configuration loadMandatoryResource(String resource)
+      throws FileNotFoundException {
+    Configuration conf = new Configuration(false);
+    URL resURL = getResourceUrl(resource);
+    if (resURL != null) {
+      log.debug("loaded resources from {}", resURL);
+      conf.addResource(resource);
+    } else {
+      throw new FileNotFoundException(resource);
+    }
+    return conf;
+  }
+
+  /**
+   * Propagate a property from a source to a dest config, with a best-effort
+   * attempt at propagating the origin.
+   * If the 
+   * @param dest destination
+   * @param src source
+   * @param key key to try to copy
+   * @return true if the key was found and propagated
+   */
+  public static boolean propagate(Configuration dest,
+                                  Configuration src,
+                                  String key) {
+    String val = src.get(key);
+    if (val != null) {
+      String[] origin = src.getPropertySources(key);
+      if (origin != null && origin.length > 0) {
+        dest.set(key, val, origin[0]);
+      } else {
+        dest.set(key, val);
+        return true;
+      }
+    }
+    return false;
+  }
+
+
+  /**
+   * Take a configuration, return a hash map
+   * @param conf conf
+   * @return hash map
+   */
+  public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
+    Map<String, String> map = new HashMap<String, String>();
+    return SliderUtils.mergeEntries(map, conf);
+  }
+
+  /**
+   * This goes through the keyset of one configuration and retrieves each value
+   * from a value source -a different or the same configuration. This triggers
+   * the property resolution process of the value, resolving any variables against
+   * in-config or inherited configurations
+   * @param keysource source of keys
+   * @param valuesource the source of values
+   * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
+   */
+  public static Configuration resolveConfiguration(
+      Iterable<Map.Entry<String, String>> keysource,
+      Configuration valuesource) {
+    Configuration result = new Configuration(false);
+    for (Map.Entry<String, String> entry : keysource) {
+      String key = entry.getKey();
+      String value = valuesource.get(key);
+      Preconditions.checkState(value != null,
+          "no reference for \"%s\" in values", key);
+      result.set(key, value);
+    }
+    return result;
+  }
+
+  /**
+   * Register anything we consider deprecated
+   */
+  public static void registerDeprecatedConfigItems() {
+    Configuration.addDeprecation(
+        SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+    Configuration.addDeprecation(
+        SliderXmlConfKeys.REGISTRY_PATH,
+        RegistryConstants.KEY_REGISTRY_ZK_ROOT);
+    
+  }
+
+  /**
+   * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
+   * included
+   * @return a configuration instance
+   */
+  public static Configuration loadSliderConfiguration() {
+    Configuration conf = new Configuration();
+    conf.addResource(SliderKeys.SLIDER_XML);
+    return conf;
+  }
+
+  /**
+   * Inject the {@link SliderKeys#SLIDER_XML} resource
+   * into the configuration resources <i>of all configurations</i>.
+   * <p>
+   *   This operation is idempotent.
+   * <p>
+   * If the resource is not on the classpath, downgrades, rather than
+   * fails.
+   * @return true if the resource was found and loaded.
+   */
+  public static synchronized boolean injectSliderXMLResource() {
+    if (sliderResourceInjectionAttempted.getAndSet(true)) {
+      return sliderResourceInjected.get();
+    }
+    URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML);
+    if (resourceUrl != null) {
+      Configuration.addDefaultResource(SliderKeys.SLIDER_XML);
+      sliderResourceInjected.set(true);
+    }
+    return sliderResourceInjected.get();
+  }
+  
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message