hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [29/39] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)
Date Tue, 02 Aug 2016 16:27:10 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
new file mode 100644
index 0000000..0344305
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+  @Override
+  public List<String> split(String value) {
+    List<String> list = new ArrayList<>(1);
+    list.add(value);
+    return list;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
new file mode 100644
index 0000000..7524053
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import java.io.File;
+
+/**
+ * Launch args for create and start and anything else that can start something
+ */
+public interface LaunchArgsAccessor extends WaitTimeAccessor {
+  String getRmAddress();
+
+  File getOutputFile();
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
new file mode 100644
index 0000000..bc7e94c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+import java.io.File;
+
+/**
+ * Any launch-time args
+ */
+public class LaunchArgsDelegate extends WaitArgsDelegate implements
+                                                         LaunchArgsAccessor {
+
+
+  //TODO: do we need this?
+  @Parameter(names = ARG_RESOURCE_MANAGER,
+             description = "Resource manager hostname:port ",
+             required = false)
+  private String rmAddress;
+
+  @Override
+  public String getRmAddress() {
+    return rmAddress;
+  }
+
+  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+      description = "output file for any application report")
+  public File outputFile;
+
+  @Override
+  public File getOutputFile() {
+    return outputFile;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
new file mode 100644
index 0000000..ccb526c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+import org.apache.hadoop.fs.Path;
+
+public class PathArgumentConverter extends BaseConverter<Path> {
+
+  public PathArgumentConverter(String optionName) {
+    super(optionName);
+  }
+
+  @Override
+  public Path convert(String value) {
+    return new Path(value);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
new file mode 100644
index 0000000..f9516d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+/**
+ * Parameters sent by the Client to the AM
+ */
+public class SliderAMArgs extends CommonArgs {
+
+  SliderAMCreateAction createAction = new SliderAMCreateAction();
+
+  public SliderAMArgs(String[] args) {
+    super(args);
+  }
+
+  @Override
+  protected void addActionArguments() {
+    addActions(createAction);
+  }
+
+  public String getImage() {
+    return createAction.image;
+  }
+
+  /**
+   * This is the URI in the FS to the Slider cluster; the conf file (and any
+   * other cluster-specifics) can be picked up here
+   */
+  public String getSliderClusterURI() {
+    return createAction.sliderClusterURI;
+  }
+
+  /**
+   * Am binding is simple: there is only one action
+   */
+  @Override
+  public void applyAction() {
+    bindCoreAction(createAction);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
new file mode 100644
index 0000000..197c22b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+import java.io.File;
+
+
+@Parameters(commandNames = {SliderActions.ACTION_CREATE},
+            commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
+
+public class SliderAMCreateAction extends AbstractActionArgs implements
+                                                           LaunchArgsAccessor {
+
+
+  @Override
+  public String getActionName() {
+    return SliderActions.ACTION_CREATE;
+  }
+
+  @Parameter(names = ARG_IMAGE, description = "image", required = false)
+  public String image;
+
+  /**
+   * This is the URI in the FS to the Slider cluster; the conf file (and any
+   * other cluster-specifics) can be picked up here
+   */
+  @Parameter(names = ARG_CLUSTER_URI,
+             description = "URI to the Slider cluster", required = true)
+  public String sliderClusterURI;
+
+  @ParametersDelegate
+  LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+
+  @Override
+  public String getRmAddress() {
+    return launchArgs.getRmAddress();
+  }
+
+  @Override
+  public int getWaittime() {
+    return launchArgs.getWaittime();
+  }
+
+  @Override
+  public void setWaittime(int waittime) {
+    launchArgs.setWaittime(waittime);
+  }
+
+  @Override
+  public File getOutputFile() {
+    return launchArgs.getOutputFile();
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
new file mode 100644
index 0000000..aab7c98
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+/**
+ * Actions.
+ * Only some of these are supported by specific Slider Services; they
+ * are listed here to ensure the names are consistent
+ */
+public interface SliderActions {
+  String ACTION_AM_SUICIDE = "am-suicide";
+  String ACTION_BUILD = "build";
+  String ACTION_CLIENT = "client";
+  String ACTION_CREATE = "create";
+  String ACTION_DIAGNOSTICS = "diagnostics";
+  String ACTION_DEPENDENCY = "dependency";
+  String ACTION_UPDATE = "update";
+  String ACTION_UPGRADE = "upgrade";
+  String ACTION_DESTROY = "destroy";
+  String ACTION_ECHO = "echo";
+  String ACTION_EXISTS = "exists";
+  String ACTION_FLEX = "flex";
+  String ACTION_FREEZE = "stop";
+  String ACTION_HELP = "help";
+  String ACTION_INSTALL_KEYTAB = "install-keytab";
+  String ACTION_INSTALL_PACKAGE = "install-package";
+  String ACTION_KDIAG = "kdiag";
+  String ACTION_KEYTAB = "keytab";
+  String ACTION_KILL_CONTAINER = "kill-container";
+  String ACTION_LIST = "list";
+  String ACTION_LOOKUP = "lookup";
+  String ACTION_NODES = "nodes";
+  String ACTION_PACKAGE = "package";
+  String ACTION_PREFLIGHT = "preflight";
+  String ACTION_RECONFIGURE = "reconfigure";
+  String ACTION_REGISTRY = "registry";
+  String ACTION_RESOLVE = "resolve";
+  String ACTION_STATUS = "status";
+  String ACTION_THAW = "start";
+  String ACTION_TOKENS = "tokens";
+
+  String ACTION_VERSION = "version";
+  String DESCRIBE_ACTION_AM_SUICIDE =
+      "Tell the Slider Application Master to simulate a process failure by terminating itself";
+  String DESCRIBE_ACTION_BUILD =
+    "Build a Slider cluster specification, but do not start it";
+  String DESCRIBE_ACTION_CREATE =
+      "Create a live Slider application";
+  String DESCRIBE_ACTION_DEPENDENCY =
+      "Slider AM and agent dependency (libraries) management";
+  String DESCRIBE_ACTION_UPDATE =
+      "Update template for a Slider application";
+  String DESCRIBE_ACTION_UPGRADE =
+      "Rolling upgrade/downgrade the application to a newer/previous version";
+  String DESCRIBE_ACTION_DESTROY =
+        "Destroy a stopped Slider application";
+  String DESCRIBE_ACTION_EXISTS =
+            "Probe for an application running";
+  String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
+  String DESCRIBE_ACTION_FREEZE =
+              "Stop a running application";
+  String DESCRIBE_ACTION_GETCONF =
+                "Get the configuration of an application";
+  String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
+  String DESCRIBE_ACTION_KILL_CONTAINER =
+    "Kill a container in the application";
+  String DESCRIBE_ACTION_HELP = "Print help information";
+  String DESCRIBE_ACTION_LIST =
+                  "List running Slider applications";
+  String DESCRIBE_ACTION_LOOKUP =
+                  "look up a YARN application";
+  String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
+  String DESCRIBE_ACTION_MONITOR =
+                    "Monitor a running application";
+  String DESCRIBE_ACTION_REGISTRY =
+                      "Query the registry of a YARN application";
+  String DESCRIBE_ACTION_RESOLVE =
+                      "Resolve or list records in the YARN registry";
+  String DESCRIBE_ACTION_STATUS =
+                      "Get the status of an application";
+  String DESCRIBE_ACTION_THAW =
+                        "Start a stopped application";
+  String DESCRIBE_ACTION_VERSION =
+                        "Print the Slider version information";
+  String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." +
+  		" Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.";
+  String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages";
+  String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore";
+  String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." +
+  		" Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'.";
+  String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
+  String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
+
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
new file mode 100644
index 0000000..b0d1ebf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class URIArgumentConverter extends BaseConverter<URI> {
+
+  public URIArgumentConverter(String optionName) {
+    super(optionName);
+  }
+
+  @Override
+  public URI convert(String value) {
+    try {
+      return new URI(value);
+    } catch (URISyntaxException e) {
+      throw new RuntimeException("Cannot make a URI from " + value);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
new file mode 100644
index 0000000..8894309
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.BaseConverter;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public class URLArgumentConverter extends BaseConverter<URL> {
+  public URLArgumentConverter(String optionName) {
+    super(optionName);
+  }
+
+  @Override
+  public URL convert(String value) {
+    try {
+      return new URL(value);
+    } catch (MalformedURLException e) {
+      throw new RuntimeException("Cannot make a URL from " + value);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
new file mode 100644
index 0000000..1c27c01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+
+public class WaitArgsDelegate extends AbstractArgsDelegate implements
+                                                           WaitTimeAccessor {
+
+
+  //--wait [timeout]
+  @Parameter(names = {ARG_WAIT},
+             description = "time to wait for an action to complete")
+  public int waittime = 0;
+
+
+  @Override
+  public int getWaittime() {
+    return waittime;
+  }
+
+  @Override
+  public void setWaittime(int waittime) {
+    this.waittime = waittime;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
new file mode 100644
index 0000000..13d4d5a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+public interface WaitTimeAccessor {
+  int getWaittime();
+  void setWaittime(int waittime);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
new file mode 100644
index 0000000..6380d0c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Some general comparators
+ */
+public class Comparators {
+
+  public static class LongComparator implements Comparator<Long>, Serializable {
+    @Override
+    public int compare(Long o1, Long o2) {
+      long result = o1 - o2;
+      // need to comparisons with a diff greater than integer size
+      if (result < 0 ) {
+        return -1;
+      } else if (result > 0) {
+        return 1;
+      }
+      return 0;
+    }
+  }
+public static class InvertedLongComparator implements Comparator<Long>, Serializable {
+  private static final LongComparator inner = new LongComparator();
+    @Override
+    public int compare(Long o1, Long o2) {
+      return -inner.compare(o1, o2);
+    }
+  }
+
+
+  /**
+   * Little template class to reverse any comparitor
+   * @param <CompareType> the type that is being compared
+   */
+  public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+      Serializable {
+
+    final Comparator<CompareType> instance;
+
+    public ComparatorReverser(Comparator<CompareType> instance) {
+      this.instance = instance;
+    }
+
+    @Override
+    public int compare(CompareType first, CompareType second) {
+      return instance.compare(second, first);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
new file mode 100644
index 0000000..9db241d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -0,0 +1,658 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Methods to aid in config, both in the Configuration class and
+ * with other parts of setting up Slider-initated processes.
+ * 
+ * Some of the methods take an argument of a map iterable for their sources; this allows
+ * the same method
+ */
+public class ConfigHelper {
+  private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
+
+  private static AtomicBoolean sliderResourceInjected =
+      new AtomicBoolean(false);
+  private static AtomicBoolean sliderResourceInjectionAttempted =
+      new AtomicBoolean(false);
+  
+  /**
+   * Dump the (sorted) configuration
+   * @param conf config
+   * @return the sorted keyset
+   */
+  public static Set<String> dumpConf(Configuration conf) {
+    Set<String> keys = sortedConfigKeys(conf);
+    for (String key : keys) {
+      log.info("{}={}", key, conf.get(key));
+    }
+    return keys;
+  }
+
+  /**
+   * Take a configuration and return a sorted set
+   * @param conf config
+   * @return the sorted keyset
+
+   */
+  public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
+    TreeSet<String> sorted = new TreeSet<String>();
+    for (Map.Entry<String, String> entry : conf) {
+      sorted.add(entry.getKey());
+    }
+    return sorted;
+  }
+
+  /**
+   * Set an entire map full of values
+   *
+   * @param config config to patch
+   * @param map map of data
+   * @param origin origin data
+   */
+  public static void addConfigMap(Configuration config,
+                                  Map<String, String> map,
+                                  String origin) throws BadConfigException {
+    addConfigMap(config, map.entrySet(), origin);
+  }
+  
+  /**
+   * Set an entire map full of values
+   *
+   * @param config config to patch
+   * @param map map of data
+   * @param origin origin data
+   */
+  public static void addConfigMap(Configuration config,
+                                  Iterable<Map.Entry<String, String>> map,
+                                  String origin) throws BadConfigException {
+    for (Map.Entry<String, String> mapEntry : map) {
+      String key = mapEntry.getKey();
+      String value = mapEntry.getValue();
+      if (value == null) {
+        throw new BadConfigException("Null value for property " + key);
+      }
+      config.set(key, value, origin);
+    }
+  }
+
+
+  /**
+   * Save a config file in a destination directory on a given filesystem
+   * @param systemConf system conf used for creating filesystems
+   * @param confToSave config to save
+   * @param confdir the directory path where the file is to go
+   * @param filename the filename
+   * @return the destination path where the file was saved
+   * @throws IOException IO problems
+   */
+  public static Path saveConfig(Configuration systemConf,
+                                Configuration confToSave,
+                                Path confdir,
+                                String filename) throws IOException {
+    FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+    Path destPath = new Path(confdir, filename);
+    saveConfig(fs, destPath, confToSave);
+    return destPath;
+  }
+
+  /**
+   * Save a config
+   * @param fs filesystem
+   * @param destPath dest to save
+   * @param confToSave  config to save
+   * @throws IOException IO problems
+   */
+  public static void saveConfig(FileSystem fs,
+                                Path destPath,
+                                Configuration confToSave) throws
+                                                              IOException {
+    FSDataOutputStream fos = fs.create(destPath);
+    try {
+      confToSave.writeXml(fos);
+    } finally {
+      IOUtils.closeStream(fos);
+    }
+  }
+
+  /**
+   * Convert to an XML string
+   * @param conf configuration
+   * @return conf
+   * @throws IOException
+   */
+  public static String toXml(Configuration conf) throws IOException {
+    StringWriter writer = new StringWriter();
+    conf.writeXml(writer);
+    return writer.toString();
+  }
+  
+  /**
+   * This will load and parse a configuration to an XML document
+   * @param fs filesystem
+   * @param path path
+   * @return an XML document
+   * @throws IOException IO failure
+   */
+  public Document parseConfiguration(FileSystem fs,
+                                     Path path) throws
+                                                IOException {
+
+
+    byte[] data = loadBytes(fs, path);
+    //this is here to track down a parse issue
+    //related to configurations
+    String s = new String(data, 0, data.length);
+    log.debug("XML resource {} is \"{}\"", path, s);
+/* JDK7
+    try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+      Document document = parseConfigXML(in);
+      return document;
+    } catch (ParserConfigurationException | SAXException e) {
+      throw new IOException(e);
+    }
+*/
+    ByteArrayInputStream in= null;
+    try {
+      in = new ByteArrayInputStream(data);
+      Document document = parseConfigXML(in);
+      return document;
+    } catch (ParserConfigurationException e) {
+      throw new IOException(e);
+    } catch (SAXException e) {
+      throw new IOException(e);
+    } finally {
+      IOUtils.closeStream(in);
+    }
+  }
+
+  public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
+    int len = (int) fs.getLength(path);
+    byte[] data = new byte[len];
+    /* JDK7
+    try(FSDataInputStream in = fs.open(path)) {
+      in.readFully(0, data);
+    }
+*/
+    FSDataInputStream in = null;
+    in = fs.open(path);
+    try {
+      in.readFully(0, data);
+    } finally {
+      IOUtils.closeStream(in);
+    }
+    return data;
+  }
+
+  /**
+   * Load a configuration from ANY FS path. The normal Configuration
+   * loader only works with file:// URIs
+   * @param fs filesystem
+   * @param path path
+   * @return a loaded resource
+   * @throws IOException
+   */
+  public static Configuration loadConfiguration(FileSystem fs,
+                                                Path path) throws IOException {
+    byte[] data = loadBytes(fs, path);
+
+    ByteArrayInputStream in2;
+
+    in2 = new ByteArrayInputStream(data);
+    Configuration conf1 = new Configuration(false);
+    conf1.addResource(in2);
+    //now clone it while dropping all its sources
+    Configuration conf2   = new Configuration(false);
+    String src = path.toString();
+    for (Map.Entry<String, String> entry : conf1) {
+      String key = entry.getKey();
+      String value = entry.getValue();
+      conf2.set(key, value, src);
+    }
+    return conf2;
+  }
+
+
+  /**
+   * Generate a config file in a destination directory on the local filesystem
+   * @param confdir the directory path where the file is to go
+   * @param filename the filename
+   * @return the destination path
+   */
+  public static File saveConfig(Configuration generatingConf,
+                                    File confdir,
+                                    String filename) throws IOException {
+
+
+    File destPath = new File(confdir, filename);
+    OutputStream fos = new FileOutputStream(destPath);
+    try {
+      generatingConf.writeXml(fos);
+    } finally {
+      IOUtils.closeStream(fos);
+    }
+    return destPath;
+  }
+
+  /**
+   * Parse an XML Hadoop configuration into an XML document. x-include
+   * is supported, but as the location isn't passed in, relative
+   * URIs are out.
+   * @param in instream
+   * @return a document
+   * @throws ParserConfigurationException parser feature problems
+   * @throws IOException IO problems
+   * @throws SAXException XML is invalid
+   */
+  public static Document parseConfigXML(InputStream in) throws
+                                               ParserConfigurationException,
+                                               IOException,
+                                               SAXException {
+    DocumentBuilderFactory docBuilderFactory
+      = DocumentBuilderFactory.newInstance();
+    //ignore all comments inside the xml file
+    docBuilderFactory.setIgnoringComments(true);
+
+    //allow includes in the xml file
+    docBuilderFactory.setNamespaceAware(true);
+    docBuilderFactory.setXIncludeAware(true);
+    DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
+    return builder.parse(in);
+  }
+
+  /**
+   * Load a Hadoop configuration from a local file.
+   * @param file file to load
+   * @return a configuration which hasn't actually had the load triggered
+   * yet.
+   * @throws FileNotFoundException file is not there
+   * @throws IOException any other IO problem
+   */
+  public static Configuration loadConfFromFile(File file) throws
+                                                          IOException {
+    return loadConfFromFile(file, false);
+  }
+
+  /**
+   *
+   * Load a Hadoop configuration from a local file.
+   * @param file file to load
+   * @param loadDefaults flag to indicate if the defaults should be loaded yet
+   * @return a configuration which hasn't actually had the load triggered
+   * yet.
+   * @throws FileNotFoundException file is not there
+   * @throws IOException any other IO problem
+   */
+  public static Configuration loadConfFromFile(File file,
+      boolean loadDefaults) throws IOException {
+    if (!file.exists()) {
+      throw new FileNotFoundException("File not found :"
+                                          + file.getAbsoluteFile());
+    }
+    Configuration conf = new Configuration(loadDefaults);
+    try {
+      conf.addResource(file.toURI().toURL());
+    } catch (MalformedURLException e) {
+      // should never happen...
+      throw new IOException(
+        "File " + file.toURI() + " doesn't have a valid URL");
+    }
+    return conf;
+  }
+
+  /**
+   * Add a configuration from a file to an existing configuration
+   * @param conf existing configuration
+   * @param file file to load
+   * @param overwrite flag to indicate new values should overwrite the predecessor
+   * @return the merged configuration
+   * @throws IOException
+   */
+  public static Configuration addConfigurationFile(Configuration conf,
+      File file, boolean overwrite)
+      throws IOException {
+    Configuration c2 = loadConfFromFile(file, false);
+    mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
+    return conf;
+  }
+
+  /**
+   * Add the system env variables with the given prefix (by convention, env.)
+   * @param conf existing configuration
+   * @param prefix prefix
+   */
+  public static void addEnvironmentVariables(Configuration conf, String prefix) {
+    Map<String, String> env = System.getenv();
+    for (Map.Entry<String, String> entry : env.entrySet()) {
+      conf.set(prefix + entry.getKey(),entry.getValue(), "env");
+    }
+  }
+  
+  /**
+   * looks for the config under $confdir/$templateFilename; if not there
+   * loads it from /conf/templateFile.
+   * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+   * origin to help debug what's happening
+   * @param systemConf system conf
+   * @param confdir conf dir in FS
+   * @param templateFilename filename in the confdir
+   * @param fallbackResource resource to fall back on
+   * @return loaded conf
+   * @throws IOException IO problems
+   */
+  public static Configuration loadTemplateConfiguration(Configuration systemConf,
+                                                        Path confdir,
+                                                        String templateFilename,
+                                                        String fallbackResource) throws
+                                                                         IOException {
+    FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
+
+    Path templatePath = new Path(confdir, templateFilename);
+    return loadTemplateConfiguration(fs, templatePath, fallbackResource);
+  }
+
+  /**
+   * looks for the config under $confdir/$templateFilename; if not there
+   * loads it from /conf/templateFile.
+   * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
+   * origin to help debug what's happening.
+   * @param fs Filesystem
+   * @param templatePath HDFS path for template
+   * @param fallbackResource resource to fall back on, or "" for no fallback
+   * @return loaded conf
+   * @throws IOException IO problems
+   * @throws FileNotFoundException if the path doesn't have a file and there
+   * was no fallback.
+   */
+  public static Configuration loadTemplateConfiguration(FileSystem fs,
+                                                        Path templatePath,
+                                                        String fallbackResource)
+      throws IOException {
+    Configuration conf;
+    String origin;
+    if (fs.exists(templatePath)) {
+      log.debug("Loading template configuration {}", templatePath);
+      conf = loadConfiguration(fs, templatePath);
+      origin = templatePath.toString();
+    } else {
+      if (fallbackResource.isEmpty()) {
+        throw new FileNotFoundException("No config file found at " + templatePath);
+      }
+      log.debug("Template {} not found" +
+                " -reverting to classpath resource {}", templatePath, fallbackResource);
+      conf = new Configuration(false);
+      conf.addResource(fallbackResource);
+      origin = "Resource " + fallbackResource;
+    }
+    //force a get
+    conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
+    //now set the origin
+    conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
+    return conf;
+  }
+
+
+  /**
+   * For testing: dump a configuration
+   * @param conf configuration
+   * @return listing in key=value style
+   */
+  public static String dumpConfigToString(Configuration conf) {
+    Set<String> sorted = sortedConfigKeys(conf);
+
+    StringBuilder builder = new StringBuilder();
+    for (String key : sorted) {
+
+      builder.append(key)
+             .append("=")
+             .append(conf.get(key))
+             .append("\n");
+    }
+    return builder.toString();
+  }
+
+  /**
+   * Merge in one configuration above another
+   * @param base base config
+   * @param merge one to merge. This MUST be a non-default-load config to avoid
+   * merge origin confusion
+   * @param origin description of the origin for the put operation
+   * @param overwrite flag to indicate new values should overwrite the predecessor
+   * @return the base with the merged values
+   */
+  public static Configuration mergeConfigurations(Configuration base,
+      Iterable<Map.Entry<String, String>> merge,
+      String origin,
+      boolean overwrite) {
+    for (Map.Entry<String, String> entry : merge) {
+      String key = entry.getKey();
+      if (overwrite || base.get(key) == null) {
+        base.set(key, entry.getValue(), origin);
+      }
+    }
+    return base;
+  }
+
+  /**
+   * Register a resource as a default resource.
+   * Do not attempt to use this unless you understand that the
+   * order in which default resources are loaded affects the outcome,
+   * and that subclasses of Configuration often register new default
+   * resources
+   * @param resource the resource name
+   * @return the URL or null
+   */
+  public static URL registerDefaultResource(String resource) {
+    URL resURL = getResourceUrl(resource);
+    if (resURL != null) {
+      Configuration.addDefaultResource(resource);
+    }
+    return resURL;
+  }
+
+  /**
+   * Load a configuration from a resource on this classpath.
+   * If the resource is not found, an empty configuration is returned
+   * @param resource the resource name
+   * @return the loaded configuration.
+   */
+  public static Configuration loadFromResource(String resource) {
+    Configuration conf = new Configuration(false);
+    URL resURL = getResourceUrl(resource);
+    if (resURL != null) {
+      log.debug("loaded resources from {}", resURL);
+      conf.addResource(resource);
+    } else{
+      log.debug("failed to find {} on the classpath", resource);
+    }
+    return conf;
+    
+  }
+
+  /**
+   * Get the URL to a resource, null if not on the CP
+   * @param resource resource to look for
+   * @return the URL or null
+   */
+  public static URL getResourceUrl(String resource) {
+    return ConfigHelper.class.getClassLoader()
+                                  .getResource(resource);
+  }
+
+  /**
+   * Load a resource that must be on the classpath
+   * @param resource the resource name
+   * @return the loaded configuration
+   * @throws FileNotFoundException if the resource is missing
+   */
+  public static Configuration loadMandatoryResource(String resource)
+      throws FileNotFoundException {
+    Configuration conf = new Configuration(false);
+    URL resURL = getResourceUrl(resource);
+    if (resURL != null) {
+      log.debug("loaded resources from {}", resURL);
+      conf.addResource(resource);
+    } else {
+      throw new FileNotFoundException(resource);
+    }
+    return conf;
+  }
+
+  /**
+   * Propagate a property from a source to a dest config, with a best-effort
+   * attempt at propagating the origin.
+   * If the 
+   * @param dest destination
+   * @param src source
+   * @param key key to try to copy
+   * @return true if the key was found and propagated
+   */
+  public static boolean propagate(Configuration dest,
+                                  Configuration src,
+                                  String key) {
+    String val = src.get(key);
+    if (val != null) {
+      String[] origin = src.getPropertySources(key);
+      if (origin != null && origin.length > 0) {
+        dest.set(key, val, origin[0]);
+      } else {
+        dest.set(key, val);
+        return true;
+      }
+    }
+    return false;
+  }
+
+
+  /**
+   * Take a configuration, return a hash map
+   * @param conf conf
+   * @return hash map
+   */
+  public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
+    Map<String, String> map = new HashMap<String, String>();
+    return SliderUtils.mergeEntries(map, conf);
+  }
+
+  /**
+   * This goes through the keyset of one configuration and retrieves each value
+   * from a value source -a different or the same configuration. This triggers
+   * the property resolution process of the value, resolving any variables against
+   * in-config or inherited configurations
+   * @param keysource source of keys
+   * @param valuesource the source of values
+   * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
+   */
+  public static Configuration resolveConfiguration(
+      Iterable<Map.Entry<String, String>> keysource,
+      Configuration valuesource) {
+    Configuration result = new Configuration(false);
+    for (Map.Entry<String, String> entry : keysource) {
+      String key = entry.getKey();
+      String value = valuesource.get(key);
+      Preconditions.checkState(value != null,
+          "no reference for \"%s\" in values", key);
+      result.set(key, value);
+    }
+    return result;
+  }
+
+  /**
+   * Register anything we consider deprecated
+   */
+  public static void registerDeprecatedConfigItems() {
+    Configuration.addDeprecation(
+        SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+    Configuration.addDeprecation(
+        SliderXmlConfKeys.REGISTRY_PATH,
+        RegistryConstants.KEY_REGISTRY_ZK_ROOT);
+    
+  }
+
+  /**
+   * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
+   * included
+   * @return a configuration instance
+   */
+  public static Configuration loadSliderConfiguration() {
+    Configuration conf = new Configuration();
+    conf.addResource(SliderKeys.SLIDER_XML);
+    return conf;
+  }
+
+  /**
+   * Inject the {@link SliderKeys#SLIDER_XML} resource
+   * into the configuration resources <i>of all configurations</i>.
+   * <p>
+   *   This operation is idempotent.
+   * <p>
+   * If the resource is not on the classpath, downgrades, rather than
+   * fails.
+   * @return true if the resource was found and loaded.
+   */
+  public static synchronized boolean injectSliderXMLResource() {
+    if (sliderResourceInjectionAttempted.getAndSet(true)) {
+      return sliderResourceInjected.get();
+    }
+    URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML);
+    if (resourceUrl != null) {
+      Configuration.addDefaultResource(SliderKeys.SLIDER_XML);
+      sliderResourceInjected.set(true);
+    }
+    return sliderResourceInjected.get();
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
new file mode 100644
index 0000000..6a02367
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -0,0 +1,831 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
+import org.apache.slider.core.persist.Filenames;
+import org.apache.slider.core.persist.InstancePaths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
+import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
+
+public class CoreFileSystem {
+  private static final Logger
+    log = LoggerFactory.getLogger(CoreFileSystem.class);
+
+  protected final FileSystem fileSystem;
+  protected final Configuration configuration;
+
+  public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
+    Preconditions.checkNotNull(fileSystem,
+                               "Cannot create a CoreFileSystem with a null FileSystem");
+    Preconditions.checkNotNull(configuration,
+                               "Cannot create a CoreFileSystem with a null Configuration");
+    this.fileSystem = fileSystem;
+    this.configuration = configuration;
+  }
+
+  public CoreFileSystem(Configuration configuration) throws IOException {
+    Preconditions.checkNotNull(configuration,
+                               "Cannot create a CoreFileSystem with a null Configuration");
+    this.fileSystem = FileSystem.get(configuration);
+    this.configuration = fileSystem.getConf();
+  }
+  
+  /**
+   * Get the temp path for this cluster
+   * @param clustername name of the cluster
+   * @return path for temp files (is not purged)
+   */
+  public Path getTempPathForCluster(String clustername) {
+    Path clusterDir = buildClusterDirPath(clustername);
+    return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX);
+  }
+
+  /**
+   * Returns the underlying FileSystem for this object.
+   *
+   * @return filesystem
+   */
+  public FileSystem getFileSystem() {
+    return fileSystem;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+      new StringBuilder("CoreFileSystem{");
+    sb.append("fileSystem=").append(fileSystem.getUri());
+    sb.append('}');
+    return sb.toString();
+  }
+
+  /**
+   * Build up the path string for a cluster instance -no attempt to
+   * create the directory is made
+   *
+   * @param clustername name of the cluster
+   * @return the path for persistent data
+   */
+  public Path buildClusterDirPath(String clustername) {
+    Preconditions.checkNotNull(clustername);
+    Path path = getBaseApplicationPath();
+    return new Path(path, SliderKeys.CLUSTER_DIRECTORY + "/" + clustername);
+  }
+
+  /**
+   * Build up the path string for app def folder -no attempt to
+   * create the directory is made
+   *
+   * @param clustername name of the cluster
+   * @return the path for persistent data
+   */
+  public Path buildAppDefDirPath(String clustername) {
+    Path path = buildClusterDirPath(clustername);
+    return new Path(path, SliderKeys.APP_DEF_DIR);
+  }
+
+  /**
+   * Build up the path string for addon folder -no attempt to
+   * create the directory is made
+   *
+   * @param clustername name of the cluster
+   * @return the path for persistent data
+   */
+  public Path buildAddonDirPath(String clustername, String addonId) {
+    Preconditions.checkNotNull(addonId);
+    Path path = buildClusterDirPath(clustername);
+    return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId);
+  }
+
+  /**
+   * Build up the path string for package install location -no attempt to
+   * create the directory is made
+   *
+   * @return the path for persistent app package
+   */
+  public Path buildPackageDirPath(String packageName, String packageVersion) {
+    Preconditions.checkNotNull(packageName);
+    Path path = getBaseApplicationPath();
+    path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
+    if (SliderUtils.isSet(packageVersion)) {
+      path = new Path(path, packageVersion);
+    }
+    return path;
+  }
+
+  /**
+   * Build up the path string for package install location -no attempt to
+   * create the directory is made
+   *
+   * @return the path for persistent app package
+   */
+  public Path buildClusterSecurityDirPath(String clusterName) {
+    Preconditions.checkNotNull(clusterName);
+    Path path = buildClusterDirPath(clusterName);
+    return new Path(path, SliderKeys.SECURITY_DIR);
+  }
+
+  /**
+   * Build up the path string for keytab install location -no attempt to
+   * create the directory is made
+   *
+   * @return the path for keytab
+   */
+  public Path buildKeytabInstallationDirPath(String keytabFolder) {
+    Preconditions.checkNotNull(keytabFolder);
+    Path path = getBaseApplicationPath();
+    return new Path(path, SliderKeys.KEYTAB_DIR + "/" + keytabFolder);
+  }
+
+  /**
+   * Build up the path string for keytab install location -no attempt to
+   * create the directory is made
+   *
+   * @return the path for keytab installation location
+   */
+  public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
+    Path homePath = getHomeDirectory();
+    Path baseKeytabDir;
+    if (keytabDir != null) {
+      baseKeytabDir = new Path(homePath, keytabDir);
+    } else {
+      baseKeytabDir = new Path(buildClusterDirPath(clusterName),
+                               SliderKeys.KEYTAB_DIR);
+    }
+    return keytabName == null ? baseKeytabDir :
+        new Path(baseKeytabDir, keytabName);
+  }
+
+  /**
+   * Create the Slider cluster path for a named cluster and all its subdirs
+   * This is a directory; a mkdirs() operation is executed
+   * to ensure that it is there.
+   *
+   * @param clustername name of the cluster
+   * @return the path to the cluster directory
+   * @throws java.io.IOException                      trouble
+   * @throws SliderException slider-specific exceptions
+   */
+  public Path createClusterDirectories(String clustername, Configuration conf)
+      throws IOException, SliderException {
+
+
+    Path clusterDirectory = buildClusterDirPath(clustername);
+    InstancePaths instancePaths = new InstancePaths(clusterDirectory);
+    createClusterDirectories(instancePaths);
+    return clusterDirectory;
+  }
+  
+  /**
+   * Create the Slider cluster path for a named cluster and all its subdirs
+   * This is a directory; a mkdirs() operation is executed
+   * to ensure that it is there.
+   *
+   * @param instancePaths instance paths
+   * @throws IOException trouble
+   * @throws SliderException slider-specific exceptions
+   */
+  public void createClusterDirectories(InstancePaths instancePaths) throws
+      IOException, SliderException {
+    Path instanceDir = instancePaths.instanceDir;
+
+    verifyDirectoryNonexistent(instanceDir);
+    FsPermission clusterPerms = getInstanceDirectoryPermissions();
+    createWithPermissions(instanceDir, clusterPerms);
+    createWithPermissions(instancePaths.snapshotConfPath, clusterPerms);
+    createWithPermissions(instancePaths.generatedConfPath, clusterPerms);
+    createWithPermissions(instancePaths.historyPath, clusterPerms);
+    createWithPermissions(instancePaths.tmpPathAM, clusterPerms);
+
+    // Data Directory
+    String dataOpts =
+      configuration.get(SliderXmlConfKeys.DATA_DIRECTORY_PERMISSIONS,
+               SliderXmlConfKeys.DEFAULT_DATA_DIRECTORY_PERMISSIONS);
+    log.debug("Setting data directory permissions to {}", dataOpts);
+    createWithPermissions(instancePaths.dataPath, new FsPermission(dataOpts));
+
+  }
+
+  /**
+   * Create a directory with the given permissions.
+   *
+   * @param dir          directory
+   * @param clusterPerms cluster permissions
+   * @throws IOException  IO problem
+   * @throws BadClusterStateException any cluster state problem
+   */
+  public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
+          IOException,
+          BadClusterStateException {
+    if (fileSystem.isFile(dir)) {
+      // HADOOP-9361 shows some filesystems don't correctly fail here
+      throw new BadClusterStateException(
+              "Cannot create a directory over a file %s", dir);
+    }
+    log.debug("mkdir {} with perms {}", dir, clusterPerms);
+    //no mask whatoever
+    fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
+    fileSystem.mkdirs(dir, clusterPerms);
+    //and force set it anyway just to make sure
+    fileSystem.setPermission(dir, clusterPerms);
+  }
+
+  /**
+   * Get the permissions of a path
+   *
+   * @param path path to check
+   * @return the permissions
+   * @throws IOException any IO problem (including file not found)
+   */
+  public FsPermission getPathPermissions(Path path) throws IOException {
+    FileStatus status = fileSystem.getFileStatus(path);
+    return status.getPermission();
+  }
+
+  public FsPermission getInstanceDirectoryPermissions() {
+    String clusterDirPermsOct =
+      configuration.get(CLUSTER_DIRECTORY_PERMISSIONS,
+                        DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+    return new FsPermission(clusterDirPermsOct);
+  }
+
+  /**
+   * Verify that the cluster directory is not present
+   *
+   * @param clustername      name of the cluster
+   * @param clusterDirectory actual directory to look for
+   * @throws IOException trouble with FS
+   * @throws SliderException If the directory exists
+   */
+  public void verifyClusterDirectoryNonexistent(String clustername,
+                                                Path clusterDirectory)
+      throws IOException, SliderException {
+    if (fileSystem.exists(clusterDirectory)) {
+      throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+              ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
+              clusterDirectory);
+    }
+  }
+  /**
+   * Verify that the given directory is not present
+   *
+   * @param clusterDirectory actual directory to look for
+   * @throws IOException    trouble with FS
+   * @throws SliderException If the directory exists
+   */
+  public void verifyDirectoryNonexistent(Path clusterDirectory) throws
+          IOException,
+      SliderException {
+    if (fileSystem.exists(clusterDirectory)) {
+      
+      log.error("Dir {} exists: {}",
+                clusterDirectory,
+                listFSDir(clusterDirectory));
+      throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+              ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
+              clusterDirectory);
+    }
+  }
+
+  /**
+   * Verify that a user has write access to a directory.
+   * It does this by creating then deleting a temp file
+   *
+   * @param dirPath actual directory to look for
+   * @throws FileNotFoundException file not found
+   * @throws IOException  trouble with FS
+   * @throws BadClusterStateException if the directory is not writeable
+   */
+  public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
+      SliderException {
+    verifyPathExists(dirPath);
+    Path tempFile = new Path(dirPath, "tmp-file-for-checks");
+    try {
+      FSDataOutputStream out ;
+      out = fileSystem.create(tempFile, true);
+      IOUtils.closeStream(out);
+      fileSystem.delete(tempFile, false);
+    } catch (IOException e) {
+      log.warn("Failed to create file {}: {}", tempFile, e);
+      throw new BadClusterStateException(e,
+              "Unable to write to directory %s : %s", dirPath, e.toString());
+    }
+  }
+
+  /**
+   * Verify that a path exists
+   * @param path path to check
+   * @throws FileNotFoundException file not found
+   * @throws IOException  trouble with FS
+   */
+  public void verifyPathExists(Path path) throws IOException {
+    if (!fileSystem.exists(path)) {
+      throw new FileNotFoundException(path.toString());
+    }
+  }
+
+  /**
+   * Verify that a path exists
+   * @param path path to check
+   * @throws FileNotFoundException file not found or is not a file
+   * @throws IOException  trouble with FS
+   */
+  public void verifyFileExists(Path path) throws IOException {
+    FileStatus status = fileSystem.getFileStatus(path);
+
+    if (!status.isFile()) {
+      throw new FileNotFoundException("Not a file: " + path.toString());
+    }
+  }
+
+  /**
+   * Given a path, check if it exists and is a file
+   * 
+   * @param path
+   *          absolute path to the file to check
+   * @returns true if and only if path exists and is a file, false for all other
+   *          reasons including if file check throws IOException
+   */
+  public boolean isFile(Path path) {
+    boolean isFile = false;
+    try {
+      FileStatus status = fileSystem.getFileStatus(path);
+      if (status.isFile()) {
+        isFile = true;
+      }
+    } catch (IOException e) {
+      // ignore, isFile is already set to false
+    }
+    return isFile;
+  }
+
+  /**
+   * Verify that a file exists in the zip file given by path
+   * @param path path to zip file
+   * @param file file expected to be in zip
+   * @throws FileNotFoundException file not found or is not a zip file
+   * @throws IOException  trouble with FS
+   */
+  public void verifyFileExistsInZip(Path path, String file) throws IOException {
+    fileSystem.copyToLocalFile(path, new Path("/tmp"));
+    File dst = new File((new Path("/tmp", path.getName())).toString());
+    Enumeration<? extends ZipEntry> entries;
+    ZipFile zipFile = new ZipFile(dst);
+    boolean found = false;
+
+    try {
+      entries = zipFile.entries();
+      while (entries.hasMoreElements()) {
+        ZipEntry entry = entries.nextElement();
+        String nm = entry.getName();
+        if (nm.endsWith(file)) {
+          found = true;
+          break;
+        }
+      }
+    } finally {
+      zipFile.close();
+    }
+    dst.delete();
+    if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
+    log.info("Verification of " + path + " passed");
+  }
+  /**
+   * Create the application-instance specific temporary directory
+   * in the DFS
+   *
+   * @param clustername name of the cluster
+   * @param subdir       application ID
+   * @return the path; this directory will already have been created
+   */
+  public Path createAppInstanceTempPath(String clustername, String subdir)
+      throws IOException {
+    Path tmp = getTempPathForCluster(clustername);
+    Path instancePath = new Path(tmp, subdir);
+    fileSystem.mkdirs(instancePath);
+    return instancePath;
+  }
+
+  /**
+   * Create the application-instance specific temporary directory
+   * in the DFS
+   *
+   * @param clustername name of the cluster
+   * @return the path; this directory will already have been deleted
+   */
+  public Path purgeAppInstanceTempFiles(String clustername) throws
+          IOException {
+    Path tmp = getTempPathForCluster(clustername);
+    fileSystem.delete(tmp, true);
+    return tmp;
+  }
+
+  /**
+   * Get the base path
+   *
+   * @return the base path optionally configured by 
+   * {@link SliderXmlConfKeys#KEY_SLIDER_BASE_PATH}
+   */
+  public Path getBaseApplicationPath() {
+    String configuredBasePath = configuration.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH);
+    return configuredBasePath != null ? new Path(configuredBasePath) :
+           new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY);
+  }
+
+  /**
+   * Get slider dependency parent dir in HDFS
+   * 
+   * @return the parent dir path of slider.tar.gz in HDFS
+   */
+  public Path getDependencyPath() {
+    String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR
+        + SliderKeys.SLIDER_DEPENDENCY_DIR
+        : SliderKeys.SLIDER_DEPENDENCY_DIR;
+    Path dependencyPath = new Path(String.format(parentDir,
+        SliderUtils.getSliderVersion()));
+    return dependencyPath;
+  }
+
+  /**
+   * Get slider.tar.gz absolute filepath in HDFS
+   * 
+   * @return the absolute path to slider.tar.gz in HDFS
+   */
+  public Path getDependencyTarGzip() {
+    Path dependencyLibAmPath = getDependencyPath();
+    Path dependencyLibTarGzip = new Path(
+        dependencyLibAmPath.toUri().toString(),
+        SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME
+            + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
+    return dependencyLibTarGzip;
+  }
+
+  public Path getHomeDirectory() {
+    return fileSystem.getHomeDirectory();
+  }
+
+  public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
+                                   Path imagePath) throws IOException {
+    if (imagePath != null) {
+      LocalResource resource = createAmResource(imagePath,
+          LocalResourceType.ARCHIVE);
+      localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
+                                   String imagePath) throws IOException {
+    
+    return imagePath != null &&
+           maybeAddImagePath(localResources, new Path(imagePath));
+  }
+  
+  
+  
+
+  /**
+   * Create an AM resource from the
+   *
+   * @param destPath     dest path in filesystem
+   * @param resourceType resource type
+   * @return the resource set up wih application-level visibility and the
+   * timestamp & size set from the file stats.
+   */
+  public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
+    FileStatus destStatus = fileSystem.getFileStatus(destPath);
+    LocalResource amResource = Records.newRecord(LocalResource.class);
+    amResource.setType(resourceType);
+    // Set visibility of the resource
+    // Setting to most private option
+    amResource.setVisibility(LocalResourceVisibility.APPLICATION);
+    // Set the resource to be copied over
+    amResource.setResource(ConverterUtils.getYarnUrlFromPath(fileSystem
+        .resolvePath(destStatus.getPath())));
+    // Set timestamp and length of file so that the framework
+    // can do basic sanity checks for the local resource
+    // after it has been copied over to ensure it is the same
+    // resource the client intended to use with the application
+    amResource.setTimestamp(destStatus.getModificationTime());
+    amResource.setSize(destStatus.getLen());
+    return amResource;
+  }
+
+  /**
+   * Register all files under a fs path as a directory to push out
+   *
+   * @param srcDir          src dir
+   * @param destRelativeDir dest dir (no trailing /)
+   * @return the map of entries
+   */
+  public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
+    //now register each of the files in the directory to be
+    //copied to the destination
+    FileStatus[] fileset = fileSystem.listStatus(srcDir);
+    Map<String, LocalResource> localResources =
+            new HashMap<String, LocalResource>(fileset.length);
+    for (FileStatus entry : fileset) {
+
+      LocalResource resource = createAmResource(entry.getPath(),
+              LocalResourceType.FILE);
+      String relativePath = destRelativeDir + "/" + entry.getPath().getName();
+      localResources.put(relativePath, resource);
+    }
+    return localResources;
+  }
+
+  /**
+   * Submit a JAR containing a specific class, returning
+   * the resource to be mapped in
+   *
+   * @param clazz   class to look for
+   * @param subdir  subdirectory (expected to end in a "/")
+   * @param jarName <i>At the destination</i>
+   * @return the local resource ref
+   * @throws IOException trouble copying to HDFS
+   */
+  public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
+          throws IOException, SliderException {
+    File localFile = SliderUtils.findContainingJarOrFail(clazz);
+    return submitFile(localFile, tempPath, subdir, jarName);
+  }
+
+  /**
+   * Submit a local file to the filesystem references by the instance's cluster
+   * filesystem
+   *
+   * @param localFile    filename
+   * @param subdir       subdirectory (expected to end in a "/")
+   * @param destFileName destination filename
+   * @return the local resource ref
+   * @throws IOException trouble copying to HDFS
+   */
+  public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
+      throws IOException {
+    Path src = new Path(localFile.toString());
+    Path subdirPath = new Path(tempPath, subdir);
+    fileSystem.mkdirs(subdirPath);
+    Path destPath = new Path(subdirPath, destFileName);
+    log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
+
+    fileSystem.copyFromLocalFile(false, true, src, destPath);
+
+    // Set the type of resource - file or archive
+    // archives are untarred at destination
+    // we don't need the jar file to be untarred for now
+    return createAmResource(destPath, LocalResourceType.FILE);
+  }
+
+  /**
+   * Submit the AM tar.gz resource referenced by the instance's cluster
+   * filesystem. Also, update the providerResources object with the new
+   * resource.
+   * 
+   * @param providerResources
+   *          the provider resource map to be updated
+   * @throws IOException
+   *           trouble copying to HDFS
+   */
+  public void submitTarGzipAndUpdate(
+      Map<String, LocalResource> providerResources) throws IOException,
+      BadClusterStateException {
+    Path dependencyLibTarGzip = getDependencyTarGzip();
+    LocalResource lc = createAmResource(dependencyLibTarGzip,
+        LocalResourceType.ARCHIVE);
+    providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc);
+  }
+
+  /**
+   * Copy local file(s) to destination HDFS directory. If {@code localPath} is a
+   * local directory then all files matching the {@code filenameFilter}
+   * (optional) are copied, otherwise {@code filenameFilter} is ignored.
+   * 
+   * @param localPath
+   *          a local file or directory path
+   * @param filenameFilter
+   *          if {@code localPath} is a directory then filenameFilter is used as
+   *          a filter (if specified)
+   * @param destDir
+   *          the destination HDFS directory where the file(s) should be copied
+   * @param fp
+   *          file permissions of all the directories and files that will be
+   *          created in this api
+   * @throws IOException
+   */
+  public void copyLocalFilesToHdfs(File localPath,
+      FilenameFilter filenameFilter, Path destDir, FsPermission fp)
+      throws IOException {
+    if (localPath == null || destDir == null) {
+      throw new IOException("Either localPath or destDir is null");
+    }
+    fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
+        "000");
+    fileSystem.mkdirs(destDir, fp);
+    if (localPath.isDirectory()) {
+      // copy all local files under localPath to destDir (honoring filename
+      // filter if provided
+      File[] localFiles = localPath.listFiles(filenameFilter);
+      Path[] localFilePaths = new Path[localFiles.length];
+      int i = 0;
+      for (File localFile : localFiles) {
+        localFilePaths[i++] = new Path(localFile.getPath());
+      }
+      log.info("Copying {} files from {} to {}", i, localPath.toURI(),
+          destDir.toUri());
+      fileSystem.copyFromLocalFile(false, true, localFilePaths, destDir);
+    } else {
+      log.info("Copying file {} to {}", localPath.toURI(), destDir.toUri());
+      fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
+          destDir);
+    }
+    // set permissions for all the files created in the destDir
+    fileSystem.setPermission(destDir, fp);
+  }
+
+  public void copyLocalFileToHdfs(File localPath,
+      Path destPath, FsPermission fp)
+      throws IOException {
+    if (localPath == null || destPath == null) {
+      throw new IOException("Either localPath or destPath is null");
+    }
+    fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
+        "000");
+    fileSystem.mkdirs(destPath.getParent(), fp);
+    log.info("Copying file {} to {}", localPath.toURI(), destPath.toUri());
+    
+    fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
+        destPath);
+    // set file permissions of the destPath
+    fileSystem.setPermission(destPath, fp);
+  }
+
+  /**
+   * list entries in a filesystem directory
+   *
+   * @param path directory
+   * @return a listing, one to a line
+   * @throws IOException
+   */
+  public String listFSDir(Path path) throws IOException {
+    FileStatus[] stats = fileSystem.listStatus(path);
+    StringBuilder builder = new StringBuilder();
+    for (FileStatus stat : stats) {
+      builder.append(stat.getPath().toString())
+              .append("\t")
+              .append(stat.getLen())
+              .append("\n");
+    }
+    return builder.toString();
+  }
+
+  /**
+   * List all application instances persisted for this user, giving the 
+   * path. The instance name is the last element in the path
+   * @return a possibly empty map of application instance names to paths
+   */
+  public Map<String, Path> listPersistentInstances() throws IOException {
+    FileSystem fs = getFileSystem();
+    Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY);
+    log.debug("Looking for all persisted application at {}", path.toString());
+    if (!fs.exists(path)) {
+      // special case: no instances have ever been created
+      return new HashMap<String, Path>(0);
+    }
+    FileStatus[] statuses = fs.listStatus(path);
+    Map<String, Path> instances = new HashMap<String, Path>(statuses.length);
+
+    // enum the child entries
+    for (FileStatus status : statuses) {
+      if (status.isDirectory()) {
+        // for directories, look for an internal.json underneath
+        Path child = status.getPath();
+        Path internalJson = new Path(child, Filenames.INTERNAL);
+        if (fs.exists(internalJson)) {
+          // success => this is an instance
+          instances.put(child.getName(), child);
+        } else {
+          log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.",
+                   child.toString());
+        }
+      }
+    }
+    return instances;
+  }
+
+  public void touch(Path path, boolean overwrite) throws IOException {
+    FSDataOutputStream out = fileSystem.create(path, overwrite);
+    out.close();
+  }
+
+  public void cat(Path path, boolean overwrite, String data) throws IOException {
+    FSDataOutputStream out = fileSystem.create(path, overwrite);
+    byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
+    out.write(bytes);
+    out.close();
+  }
+
+  /**
+   * Create a path that must exist in the cluster fs
+   * @param uri uri to create
+   * @return the path
+   * @throws SliderException if the path does not exist
+   */
+  public Path createPathThatMustExist(String uri) throws
+      SliderException, IOException {
+    Preconditions.checkNotNull(uri);
+    Path path = new Path(uri);
+    verifyPathExists(path);
+    return path;
+  }
+
+  /**
+   * Locate an application conf json in the FS. This includes a check to verify
+   * that the file is there.
+   *
+   * @param clustername name of the cluster
+   * @return the path to the spec.
+   * @throws IOException IO problems
+   * @throws SliderException if the path isn't there
+   */
+  public Path locateInstanceDefinition(String clustername) throws IOException,
+      SliderException {
+    Path clusterDirectory = buildClusterDirPath(clustername);
+    Path appConfPath =
+            new Path(clusterDirectory, Filenames.APPCONF);
+    verifyClusterSpecExists(clustername, appConfPath);
+    return appConfPath;
+  }
+
+  /**
+   * Verify that a cluster specification exists
+   * @param clustername name of the cluster (For errors only)
+   * @param clusterSpecPath cluster specification path
+   * @throws IOException IO problems
+   * @throws SliderException if the cluster specification is not present
+   */
+  public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
+      throws IOException,
+      SliderException {
+    if (!fileSystem.isFile(clusterSpecPath)) {
+      log.debug("Missing specification file {}", clusterSpecPath);
+      throw UnknownApplicationInstanceException.unknownInstance(
+          clustername + "\n (definition not found at " + clusterSpecPath);
+    }
+  }
+
+  
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message