hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [27/50] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)
Date Tue, 11 Oct 2016 20:37:31 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
new file mode 100644
index 0000000..1bdfb9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Client code for interacting with a list of service instances.
+ * The initial logic just enumerates service instances in the YARN RM
+ */
+public class YarnAppListClient {
+
+  private final SliderYarnClientImpl yarnClient;
+  private final String username;
+  private final Configuration conf;
+  private static final Logger log =
+      LoggerFactory.getLogger(YarnAppListClient.class);
+
+  public YarnAppListClient(SliderYarnClientImpl yarnClient,
+      String username,
+      Configuration conf) {
+
+    Preconditions.checkArgument(yarnClient != null,
+        "yarn client is null: is app inited?");
+    Preconditions.checkArgument(username != null,
+        "username is null");
+    Preconditions.checkArgument(conf != null,
+        "conf parameter is null");
+    this.yarnClient = yarnClient;
+    this.username = username;
+    this.conf = conf;
+  }
+
+  /**
+   * find all live instances of a specific app -if there is more than one 
+   * in the cluster, this returns them all. State should be running or earlier
+   * in the lifecycle
+   * @param appname application name
+   * @return the list of all matching application instances
+   */
+  public List<ApplicationReport> findAllLiveInstances(String appname)
+    throws YarnException, IOException {
+    return yarnClient.findAllLiveInstances(username, appname);
+  }
+
+
+  /**
+   * Find an instance of a application belong to the current user
+   * @param appname application name
+   * @return the app report or null if none is found
+   * @throws YarnException YARN issues
+   * @throws IOException IO problems
+   */
+  public ApplicationReport findInstance(String appname) throws
+                                                        YarnException,
+                                                        IOException {
+    List<ApplicationReport> instances = listInstances(null);
+    return yarnClient.findClusterInInstanceList(instances, appname);
+  }
+
+  /**
+   * List instances belonging to the specific user
+   * @return a possibly empty list of AMs
+   */
+  public List<ApplicationReport> listInstances()
+      throws YarnException, IOException {
+    return listInstances(null);
+  }
+
+  /**
+   * List instances belonging to a specific user
+   * @return a possibly empty list of AMs
+   * @param user user if not the default. null means default, "" means all users, 
+   * otherwise it is the name of a user
+   */
+  public List<ApplicationReport> listInstances(String user)
+      throws YarnException, IOException {
+    String listUser = user == null ? username : user;
+    return yarnClient.listDeployedInstances(listUser);
+  }
+
+  /**
+   * Enumerate slider instances for the current user, and the
+   * most recent app report, where available.
+   * @param listOnlyInState boolean to indicate that the instances should
+   * only include those in a YARN state
+   * <code> minAppState &lt;= currentState &lt;= maxAppState </code>
+   * 
+   * @param minAppState minimum application state to include in enumeration.
+   * @param maxAppState maximum application state to include
+   * @return a map of application instance name to description
+   * @throws IOException Any IO problem
+   * @throws YarnException YARN problems
+   */
+  public Map<String, SliderInstanceDescription> enumSliderInstances(
+      boolean listOnlyInState,
+      YarnApplicationState minAppState,
+      YarnApplicationState maxAppState)
+      throws IOException, YarnException {
+
+    CoreFileSystem sliderFileSystem = new CoreFileSystem(conf);
+    Preconditions.checkArgument(!listOnlyInState || minAppState != null,
+        "null minAppState when listOnlyInState set");
+    Preconditions.checkArgument(!listOnlyInState || maxAppState != null,
+        "null maxAppState when listOnlyInState set");
+    if (!listOnlyInState) {
+      // if there's not filtering, ask for the entire range of states
+      minAppState = YarnApplicationState.NEW;
+      maxAppState = YarnApplicationState.KILLED;
+    }
+    // get the complete list of persistent instances
+    Map<String, Path> persistentInstances =
+        sliderFileSystem.listPersistentInstances();
+    Map<String, SliderInstanceDescription> descriptions =
+        new HashMap<String, SliderInstanceDescription>(persistentInstances.size());
+
+    if (persistentInstances.isEmpty()) {
+      // an empty listing is a success if no cluster was named
+      log.debug("No application instances found");
+      return descriptions;
+    }
+
+    // enum those the RM knows about
+    List<ApplicationReport> rmInstances = listInstances();
+    SliderUtils.sortApplicationsByMostRecent(rmInstances);
+    Map<String, ApplicationReport> reportMap =
+        SliderUtils.buildApplicationReportMap(rmInstances, minAppState,
+            maxAppState);
+    log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
+        persistentInstances.size(),
+        rmInstances.size(),
+        minAppState, maxAppState,
+        reportMap.size());
+
+    // at this point there is a list of all persistent instances, and
+    // a (possibly filtered) list of application reports
+
+    for (Map.Entry<String, Path> entry : persistentInstances.entrySet()) {
+      // loop through the persistent values
+      String name = entry.getKey();
+
+      // look up any report from the (possibly filtered) report set
+      ApplicationReport report = reportMap.get(name);
+      if (!listOnlyInState || report != null) {
+        // if the enum wants to filter in state, only add it if there is
+        // a report in that range. Otherwise: include all values
+        SliderInstanceDescription sid = new SliderInstanceDescription(
+            name, entry.getValue(), report);
+        descriptions.put(name, sid);
+      }
+    }
+
+    return descriptions;
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
new file mode 100644
index 0000000..ddab606
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+public enum ConfigFormat {
+
+  JSON("json"),
+  PROPERTIES("properties"),
+  XML("xml"),
+  HADOOP_XML("hadoop-xml"),
+  ENV("env"),
+  TEMPLATE("template"),
+  YAML("yaml"),
+  ;
+  ConfigFormat(String suffix) {
+    this.suffix = suffix;
+  }
+
+  private final String suffix;
+
+  public String getSuffix() {
+    return suffix;
+  }
+
+
+  @Override
+  public String toString() {
+    return suffix;
+  }
+
+  /**
+   * Get a matching format or null
+   * @param type
+   * @return the format
+   */
+  public static ConfigFormat resolve(String type) {
+    for (ConfigFormat format: values()) {
+      if (format.getSuffix().equals(type)) {
+        return format;
+      }
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
new file mode 100644
index 0000000..2e1615b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ConfigUtils {
+  public static final String TEMPLATE_FILE = "template.file";
+
+  public static String replaceProps(Map<String, String> config, String content) {
+    Map<String, String> tokens = new HashMap<>();
+    for (Entry<String, String> entry : config.entrySet()) {
+      tokens.put("${" + entry.getKey() + "}", entry.getValue());
+      tokens.put("{{" + entry.getKey() + "}}", entry.getValue());
+    }
+    String value = content;
+    for (Map.Entry<String,String> token : tokens.entrySet()) {
+      value = value.replaceAll(Pattern.quote(token.getKey()),
+          Matcher.quoteReplacement(token.getValue()));
+    }
+    return value;
+  }
+
+  public static Map<String, String> replacePropsInConfig(
+      Map<String, String> config, Map<String, String> env) {
+    Map<String, String> tokens = new HashMap<>();
+    for (Entry<String, String> entry : env.entrySet()) {
+      tokens.put("${" + entry.getKey() + "}", entry.getValue());
+    }
+    Map<String, String> newConfig = new HashMap<>();
+    for (Entry<String, String> entry : config.entrySet()) {
+      String value = entry.getValue();
+      for (Map.Entry<String,String> token : tokens.entrySet()) {
+        value = value.replaceAll(Pattern.quote(token.getKey()),
+            Matcher.quoteReplacement(token.getValue()));
+      }
+      newConfig.put(entry.getKey(), entry.getValue());
+    }
+    return newConfig;
+  }
+
+  public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+      Map<String, String> config, SliderFileSystem fileSystem,
+      String clusterName, String fileName) throws IOException {
+    if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+      return;
+    }
+    Path templateFile = null;
+    if (config.containsKey(TEMPLATE_FILE)) {
+      templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+      if (!fileSystem.isFile(templateFile)) {
+        templateFile = fileSystem.buildResourcePath(clusterName,
+            config.get(TEMPLATE_FILE));
+      }
+      if (!fileSystem.isFile(templateFile)) {
+        throw new IOException("config specified template file " + config
+            .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+      }
+    }
+    if (templateFile == null && fileName != null) {
+      templateFile = fileSystem.buildResourcePath(fileName);
+      if (!fileSystem.isFile(templateFile)) {
+        templateFile = fileSystem.buildResourcePath(clusterName,
+            fileName);
+      }
+    }
+    if (fileSystem.isFile(templateFile)) {
+      config.put("content", fileSystem.cat(templateFile));
+    } else {
+      config.put("content", "");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
new file mode 100644
index 0000000..88bac77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigurationResolver.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+public class ConfigurationResolver {
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
new file mode 100644
index 0000000..4bcf6c1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ExportEntry {
+
+  /**
+   * The value of the export
+   */
+  private String value;
+  /**
+   * The container id of the container that is responsible for the export
+   */
+  private String containerId;
+  /**
+   * Tag associated with the container - its usually an identifier different than container id
+   * that allows a soft serial id to all containers of a component - e.g. 1, 2, 3, ...
+   */
+  private String tag;
+  /**
+   * An export can be at the level of a component or an application
+   */
+  private String level;
+  /**
+   * The time when the export was updated
+   */
+  private String updatedTime;
+  /**
+   * The time when the export expires
+   */
+  private String validUntil;
+
+  public ExportEntry() {
+  }
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+
+  public String getContainerId() {
+    return containerId;
+  }
+
+  public void setContainerId(String containerId) {
+    this.containerId = containerId;
+  }
+
+  public String getTag() {
+    return tag;
+  }
+
+  public void setTag(String tag) {
+    this.tag = tag;
+  }
+
+  public String getLevel() {
+    return level;
+  }
+
+  public void setLevel(String level) {
+    this.level = level;
+  }
+  public String getUpdatedTime() {
+    return updatedTime;
+  }
+
+  public void setUpdatedTime(String updatedTime) {
+    this.updatedTime = updatedTime;
+  }
+
+  public String getValidUntil() {
+    return validUntil;
+  }
+
+  public void setValidUntil(String validUntil) {
+    this.validUntil = validUntil;
+  }
+
+  @Override
+  public String toString() {
+    return new StringBuilder("ExportEntry{").
+        append("value='").append(value).append("',").
+        append("containerId='").append(containerId).append("',").
+        append("tag='").append(tag).append("',").
+        append("level='").append(level).append("'").
+        append("updatedTime='").append(updatedTime).append("'").
+        append("validUntil='").append(validUntil).append("'").
+        append(" }").toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
new file mode 100644
index 0000000..edc129e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Represents a set of configurations for an application, component, etc.
+ * Json serialisable; accessors are synchronized
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfigSet {
+
+  private static final PatternValidator validator = new PatternValidator(
+      RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
+  
+  public Map<String, PublishedConfiguration> configurations =
+      new HashMap<>();
+
+  public PublishedConfigSet() {
+  }
+
+  /**
+   * Put a name -it will be converted to lower case before insertion.
+   * Any existing entry will be overwritten (that includes an entry
+   * with a different case in the original name)
+   * @param name name of entry
+   * @param conf configuration
+   * @throws IllegalArgumentException if not a valid name
+   */
+  public void put(String name, PublishedConfiguration conf) {
+    String name1 = name.toLowerCase(Locale.ENGLISH);
+    validateName(name1);
+    configurations.put(name1, conf);
+  }
+
+  /**
+   * Validate the name -restricting it to the set defined in 
+   * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+   * @param name name to validate
+   * @throws IllegalArgumentException if not a valid name
+   */
+  public static void validateName(String name) {
+    validator.validate(name);
+    
+  }
+
+  public PublishedConfiguration get(String name) {
+    return configurations.get(name);
+  }
+  
+  public boolean contains(String name) {
+    return configurations.containsKey(name);
+  }
+  
+  public int size() {
+    return configurations.size();
+  }
+  
+  public Set<String> keys() {
+    TreeSet<String> keys = new TreeSet<>();
+    keys.addAll(configurations.keySet());
+    return keys;
+  }
+
+  public PublishedConfigSet shallowCopy() {
+    PublishedConfigSet that = new PublishedConfigSet();
+    for (Map.Entry<String, PublishedConfiguration> entry :
+        configurations.entrySet()) {
+      that.put(entry.getKey(), entry.getValue().shallowCopy());
+    }
+    return that;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
new file mode 100644
index 0000000..50b522f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ * 
+ * The values themselves are not serialized in the external view; they have
+ * to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedConfiguration {
+
+  public String description;
+  public long updated;
+  
+  public String updatedTime;
+
+  public Map<String, String> entries = new HashMap<>();
+
+  public PublishedConfiguration() {
+  }
+
+  /**
+   * build an empty published configuration 
+   * @param description configuration description
+   */
+  public PublishedConfiguration(String description) {
+    this.description = description;
+  }
+
+  /**
+   * Build a configuration from the entries
+   * @param description configuration description
+   * @param entries entries to put
+   */
+  public PublishedConfiguration(String description,
+      Iterable<Map.Entry<String, String>> entries) {
+    this.description = description;
+    putValues(entries);
+  }
+
+  /**
+   * Build a published configuration, using the keys from keysource,
+   * but resolving the values from the value source, via Configuration.get()
+   * @param description configuration description
+   * @param keysource source of keys
+   * @param valuesource source of values
+   */
+  public PublishedConfiguration(String description,
+      Iterable<Map.Entry<String, String>> keysource,
+      Configuration valuesource) {
+    this.description = description;
+    putValues(ConfigHelper.resolveConfiguration(keysource, valuesource));
+  }
+
+  
+  /**
+   * Is the configuration empty. This means either that it has not
+   * been given any values, or it is stripped down copy set down over the
+   * wire.
+   * @return true if it is empty
+   */
+  public boolean isEmpty() {
+    return entries.isEmpty();
+  }
+
+
+  public void setUpdated(long updated) {
+    this.updated = updated;
+    this.updatedTime = new Date(updated).toString();
+  }
+
+  public long getUpdated() {
+    return updated;
+  }
+
+  /**
+   * Set the values from an iterable (this includes a Hadoop Configuration
+   * and Java properties object).
+   * Any existing value set is discarded
+   * @param entries entries to put
+   */
+  public void putValues(Iterable<Map.Entry<String, String>> entries) {
+    this.entries = new HashMap<String, String>();
+    for (Map.Entry<String, String> entry : entries) {
+      this.entries.put(entry.getKey(), entry.getValue());
+    }
+    
+  }
+
+  /**
+   * Convert to Hadoop XML
+   * @return the configuration as a Hadoop Configuratin
+   */
+  public Configuration asConfiguration() {
+    Configuration conf = new Configuration(false);
+    try {
+      ConfigHelper.addConfigMap(conf, entries, "");
+    } catch (BadConfigException e) {
+      // triggered on a null value; switch to a runtime (and discard the stack)
+      throw new RuntimeException(e.toString());
+    }
+    return conf;
+  }
+  
+  public String asConfigurationXML() throws IOException {
+    return ConfigHelper.toXml(asConfiguration());
+  }
+
+  /**
+   * Convert values to properties
+   * @return a property file
+   */
+  public Properties asProperties() {
+    Properties props = new Properties();
+    props.putAll(entries);
+    return props;
+  }
+
+  /**
+   * Return the values as json string
+   * @return the JSON representation
+   * @throws IOException marshalling failure
+   */
+  public String asJson() throws IOException {
+    ObjectMapper mapper = new ObjectMapper();
+    mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+    String json = mapper.writeValueAsString(entries);
+    return json;
+  }
+
+
+  /**
+   * This makes a copy without the nested content -so is suitable
+   * for returning as part of the list of a parent's values
+   * @return the copy
+   */
+  public PublishedConfiguration shallowCopy() {
+    PublishedConfiguration that = new PublishedConfiguration();
+    that.description = this.description;
+    that.updated = this.updated;
+    that.updatedTime = this.updatedTime;
+    return that;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("PublishedConfiguration{");
+    sb.append("description='").append(description).append('\'');
+    sb.append(" entries = ").append(entries.size());
+    sb.append('}');
+    return sb.toString();
+  }
+
+  /**
+   * Create an outputter for a given format
+   * @param format format to use
+   * @return an instance of output
+   */
+  public PublishedConfigurationOutputter createOutputter(ConfigFormat format) {
+    return PublishedConfigurationOutputter.createOutputter(format, this);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
new file mode 100644
index 0000000..9bdcfcb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.util.Properties;
+
+/**
+ * Output a published configuration
+ */
+public abstract class PublishedConfigurationOutputter {
+
+  protected final PublishedConfiguration owner;
+
+  protected PublishedConfigurationOutputter(PublishedConfiguration owner) {
+    this.owner = owner;
+  }
+
+  /**
+   * Save the config to a destination file, in the format of this outputter
+   * @param dest destination file
+   * @throws IOException
+   */
+/* JDK7
+  public void save(File dest) throws IOException {
+    try(FileOutputStream out = new FileOutputStream(dest)) {
+      save(out);
+      out.close();
+    }
+  }
+*/
+  public void save(File dest) throws IOException {
+    FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+  }
+
+  /**
+   * Save the content. The default saves the asString() value
+   * to the output stream
+   * @param out output stream
+   * @throws IOException
+   */
+  public void save(OutputStream out) throws IOException {
+    IOUtils.write(asString(), out, Charsets.UTF_8);
+  }
+  /**
+   * Convert to a string
+   * @return the string form
+   * @throws IOException
+   */
+  public abstract String asString() throws IOException;
+
+  /**
+   * Create an outputter for the chosen format
+   * @param format format enumeration
+   * @param owner owning config
+   * @return the outputter
+   */
+
+  public static PublishedConfigurationOutputter createOutputter(ConfigFormat format,
+      PublishedConfiguration owner) {
+    Preconditions.checkNotNull(owner);
+    switch (format) {
+      case XML:
+      case HADOOP_XML:
+        return new XmlOutputter(owner);
+      case PROPERTIES:
+        return new PropertiesOutputter(owner);
+      case JSON:
+        return new JsonOutputter(owner);
+      case ENV:
+        return new EnvOutputter(owner);
+      case TEMPLATE:
+        return new TemplateOutputter(owner);
+      case YAML:
+        return new YamlOutputter(owner);
+      default:
+        throw new RuntimeException("Unsupported format :" + format);
+    }
+  }
+
+  public static class XmlOutputter extends PublishedConfigurationOutputter {
+
+
+    private final Configuration configuration;
+
+    public XmlOutputter(PublishedConfiguration owner) {
+      super(owner);
+      configuration = owner.asConfiguration();
+    }
+
+    @Override
+    public void save(OutputStream out) throws IOException {
+      configuration.writeXml(out);
+    }
+
+    @Override
+    public String asString() throws IOException {
+      return ConfigHelper.toXml(configuration);
+    }
+
+    public Configuration getConfiguration() {
+      return configuration;
+    }
+  }
+
+  public static class PropertiesOutputter extends PublishedConfigurationOutputter {
+
+    private final Properties properties;
+
+    public PropertiesOutputter(PublishedConfiguration owner) {
+      super(owner);
+      properties = owner.asProperties();
+    }
+
+    @Override
+    public void save(OutputStream out) throws IOException {
+      properties.store(out, "");
+    }
+
+
+    public String asString() throws IOException {
+      StringWriter sw = new StringWriter();
+      properties.store(sw, "");
+      return sw.toString();
+    }
+  }
+
+
+  public static class JsonOutputter extends PublishedConfigurationOutputter {
+
+    public JsonOutputter(PublishedConfiguration owner) {
+      super(owner);
+    }
+
+    @Override
+    public String asString() throws IOException {
+      return owner.asJson();
+    }
+  }
+
+
+  public static class EnvOutputter extends PublishedConfigurationOutputter {
+
+    public EnvOutputter(PublishedConfiguration owner) {
+      super(owner);
+    }
+
+    @Override
+    public String asString() throws IOException {
+      if (!owner.entries.containsKey("content")) {
+        throw new IOException("Configuration has no content field and cannot " +
+            "be retrieved as type 'env'");
+      }
+      String content = owner.entries.get("content");
+      return ConfigUtils.replaceProps(owner.entries, content);
+    }
+  }
+
+  public static class TemplateOutputter extends EnvOutputter {
+    public TemplateOutputter(PublishedConfiguration owner) {
+      super(owner);
+    }
+  }
+
+  public static class YamlOutputter extends PublishedConfigurationOutputter {
+
+    private final Yaml yaml;
+
+    public YamlOutputter(PublishedConfiguration owner) {
+      super(owner);
+      DumperOptions options = new DumperOptions();
+      options.setDefaultFlowStyle(FlowStyle.BLOCK);
+      yaml = new Yaml(options);
+    }
+
+    public String asString() throws IOException {
+      return yaml.dump(owner.entries);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
new file mode 100644
index 0000000..0759b4e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * JSON-serializable description of a published key-val configuration.
+ *
+ * The values themselves are not serialized in the external view; they have to be served up by the far end
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExports {
+
+  public String description;
+  public long updated;
+  public String updatedTime;
+  public Map<String, List<ExportEntry>> entries = new HashMap<>();
+
+  public PublishedExports() {
+  }
+
+  /**
+   * build an empty published configuration
+   *
+   * @param description configuration description
+   */
+  public PublishedExports(String description) {
+    this.description = description;
+  }
+
+  /**
+   * Build a configuration from the entries
+   *
+   * @param description configuration description
+   * @param entries     entries to put
+   */
+  public PublishedExports(String description,
+                          Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+    this.description = description;
+    putValues(entries);
+  }
+
+  /**
+   * Is the configuration empty. This means either that it has not been given any values,
+   * or it is stripped down copy
+   * set down over the wire.
+   *
+   * @return true if it is empty
+   */
+  public boolean isEmpty() {
+    return entries.isEmpty();
+  }
+
+  public long getUpdated() {
+    return updated;
+  }
+
+  public void setUpdated(long updated) {
+    this.updated = updated;
+    this.updatedTime = new Date(updated).toString();
+  }
+
+  /**
+   * Set the values from an iterable (this includes a Hadoop Configuration and Java properties
+   * object). Any existing value set is discarded
+   *
+   * @param entries entries to put
+   */
+  public void putValues(Iterable<Map.Entry<String, List<ExportEntry>>> entries) {
+    this.entries = new HashMap<String, List<ExportEntry>>();
+    for (Map.Entry<String, List<ExportEntry>> entry : entries) {
+      this.entries.put(entry.getKey(), entry.getValue());
+    }
+  }
+
+  /**
+   * Return the values as json string
+   *
+   * @return the JSON form
+   *
+   * @throws IOException mapping problems
+   */
+  public String asJson() throws IOException {
+    ObjectMapper mapper = new ObjectMapper();
+    mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
+    String json = mapper.writeValueAsString(entries);
+    return json;
+  }
+
+  /**
+   * This makes a copy without the nested content -so is suitable for returning as part of the list of a parent's
+   * values
+   *
+   * @return the copy
+   */
+  public PublishedExports shallowCopy() {
+    PublishedExports that = new PublishedExports();
+    that.description = this.description;
+    that.updated = this.updated;
+    that.updatedTime = this.updatedTime;
+    return that;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("PublishedConfiguration{");
+    sb.append("description='").append(description).append('\'');
+    sb.append(" entries = ").append(entries.size());
+    sb.append('}');
+    return sb.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
new file mode 100644
index 0000000..67cb094
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsOutputter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/** Output a published configuration */
+public abstract class PublishedExportsOutputter {
+
+  protected final PublishedExports exports;
+
+  protected PublishedExportsOutputter(PublishedExports exports) {
+    this.exports = exports;
+  }
+
+  /**
+   * Create an outputter for the chosen format
+   *
+   * @param format  format enumeration
+   * @param exports owning config
+   * @return the outputter
+   */
+
+  public static PublishedExportsOutputter createOutputter(ConfigFormat format,
+                                                         PublishedExports exports) {
+    Preconditions.checkNotNull(exports);
+    switch (format) {
+      case JSON:
+        return new JsonOutputter(exports);
+      default:
+        throw new RuntimeException("Unsupported format :" + format);
+    }
+  }
+
+  public void save(File dest) throws IOException {
+    FileOutputStream out = null;
+    try {
+      out = new FileOutputStream(dest);
+      save(out);
+      out.close();
+    } finally {
+      org.apache.hadoop.io.IOUtils.closeStream(out);
+    }
+  }
+
+  /**
+   * Save the content. The default saves the asString() value to the output stream
+   *
+   * @param out output stream
+   * @throws IOException
+   */
+  public void save(OutputStream out) throws IOException {
+    IOUtils.write(asString(), out, Charsets.UTF_8);
+  }
+
+  /**
+   * Convert to a string
+   *
+   * @return the string form
+   * @throws IOException
+   */
+  public abstract String asString() throws IOException;
+
+  public static class JsonOutputter extends PublishedExportsOutputter {
+
+    public JsonOutputter(PublishedExports exports) {
+      super(exports);
+    }
+
+    @Override
+    public void save(File dest) throws IOException {
+      FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
+    }
+
+    @Override
+    public String asString() throws IOException {
+      return exports.asJson();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
new file mode 100644
index 0000000..339d3d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Represents a set of configurations for an application, component, etc.
+ * Json serialisable; accessors are synchronized
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PublishedExportsSet {
+
+  private static final PatternValidator validator = new PatternValidator(
+      RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
+  
+  public Map<String, PublishedExports> exports = new HashMap<>();
+
+  public PublishedExportsSet() {
+  }
+
+  /**
+   * Put a name -it will be converted to lower case before insertion.
+   * Any existing entry will be overwritten (that includes an entry
+   * with a different case in the original name)
+   * @param name name of entry
+   * @param export published export
+   * @throws IllegalArgumentException if not a valid name
+   */
+  public void put(String name, PublishedExports export) {
+    String name1 = name.toLowerCase(Locale.ENGLISH);
+    validateName(name1);
+    exports.put(name1, export);
+  }
+
+  /**
+   * Validate the name -restricting it to the set defined in 
+   * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+   * @param name name to validate
+   * @throws IllegalArgumentException if not a valid name
+   */
+  public static void validateName(String name) {
+    validator.validate(name);
+    
+  }
+
+  public PublishedExports get(String name) {
+    return exports.get(name);
+  }
+  
+  public boolean contains(String name) {
+    return exports.containsKey(name);
+  }
+  
+  public int size() {
+    return exports.size();
+  }
+  
+  public Set<String> keys() {
+    TreeSet<String> keys = new TreeSet<>();
+    keys.addAll(exports.keySet());
+    return keys;
+  }
+
+  public PublishedExportsSet shallowCopy() {
+    PublishedExportsSet that = new PublishedExportsSet();
+    for (Map.Entry<String, PublishedExports> entry : exports.entrySet()) {
+      that.put(entry.getKey(), entry.getValue().shallowCopy());
+    }
+    return that;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
new file mode 100644
index 0000000..120966f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.docstore;
+
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class UriMap {
+
+  public Map<String, String> uris = new HashMap<>();
+  
+  @JsonIgnore
+  public void put(String key, String value) {
+    uris.put(key, value);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
new file mode 100644
index 0000000..13ad5c5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.info;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+  public static final String MANAGEMENT_REST_API =
+      "classpath:org.apache.slider.management";
+  
+  public static final String REGISTRY_REST_API =
+      "classpath:org.apache.slider.registry";
+  
+  public static final String PUBLISHER_REST_API =
+      "classpath:org.apache.slider.publisher";
+
+  public static final String PUBLISHER_CONFIGURATIONS_API =
+      "classpath:org.apache.slider.publisher.configurations";
+
+  public static final String PUBLISHER_EXPORTS_API =
+      "classpath:org.apache.slider.publisher.exports";
+
+  public static final String PUBLISHER_DOCUMENTS_API =
+      "classpath:org.apache.slider.publisher.documents";
+
+  public static final String AGENT_SECURE_REST_API =
+      "classpath:org.apache.slider.agents.secure";
+
+  public static final String AGENT_ONEWAY_REST_API =
+      "classpath:org.apache.slider.agents.oneway";
+
+  public static final String AM_IPC_PROTOCOL =
+      "classpath:org.apache.slider.appmaster.ipc";
+
+  public static final String AM_REST_BASE =
+      "classpath:org.apache.slider.client.rest";
+
+  public static final String WEB_UI = "http://";
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
new file mode 100644
index 0000000..40fa217
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.retrieve;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.apache.slider.client.rest.BaseRestClient;
+import org.apache.slider.core.restclient.HttpVerb;
+import org.apache.slider.core.restclient.UgiJerseyBinding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+
+/**
+ * Class to retrieve artifacts from the AM's web site. This sets up
+ * the redirection and security logic properly
+ */
+public class AMWebClient {
+
+
+  private final BaseRestClient restClient;
+  private static final Logger
+      log = LoggerFactory.getLogger(AMWebClient.class);
+
+
+  public AMWebClient(Configuration conf) {
+    UgiJerseyBinding binding = new UgiJerseyBinding(conf);
+
+    restClient = new BaseRestClient(binding.createJerseyClient());
+
+  }
+
+
+  private static URLConnectionClientHandler getUrlConnectionClientHandler() {
+    return new URLConnectionClientHandler(new HttpURLConnectionFactory() {
+      @Override
+      public HttpURLConnection getHttpURLConnection(URL url)
+          throws IOException {
+        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+        if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) {
+          // is a redirect - are we changing schemes?
+          String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION);
+          String originalScheme = url.getProtocol();
+          String redirectScheme = URI.create(redirectLocation).getScheme();
+          if (!originalScheme.equals(redirectScheme)) {
+            // need to fake it out by doing redirect ourselves
+            log.info("Protocol change during redirect. Redirecting {} to URL {}",
+                     url, redirectLocation);
+            URL redirectURL = new URL(redirectLocation);
+            connection = (HttpURLConnection) redirectURL.openConnection();
+          }
+        }
+        if (connection instanceof HttpsURLConnection) {
+          log.debug("Attempting to configure HTTPS connection using client "
+                    + "configuration");
+          final SSLFactory factory;
+          final SSLSocketFactory sf;
+          final HostnameVerifier hv;
+
+          try {
+            HttpsURLConnection c = (HttpsURLConnection) connection;
+            factory = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration());
+            factory.init();
+            sf = factory.createSSLSocketFactory();
+            hv = factory.getHostnameVerifier();
+            c.setSSLSocketFactory(sf);
+            c.setHostnameVerifier(hv);
+          } catch (Exception e) {
+            log.info("Unable to configure HTTPS connection from "
+                     + "configuration.  Using JDK properties.");
+          }
+
+        }
+        return connection;
+      }
+    });
+  }
+
+  public WebResource resource(String url) {
+    return restClient.resource(url);
+  }
+
+  public BaseRestClient getRestClient() {
+    return restClient;
+  }
+
+  /**
+   * Execute the operation. Failures are raised as IOException subclasses
+   * @param method method to execute
+   * @param resource resource to work against
+   * @param c class to build
+   * @param <T> type expected
+   * @return an instance of the type T
+   * @throws IOException on any failure
+   */
+  public <T> T exec(HttpVerb method, WebResource resource, Class<T> c) throws IOException {
+    return restClient.exec(method, resource, c);
+  }
+
+  /**
+   * Execute the operation. Failures are raised as IOException subclasses
+   * @param method method to execute
+   * @param resource resource to work against
+   * @param t type to work with
+   * @param <T> type expected
+   * @return an instance of the type T
+   * @throws IOException on any failure
+   */
+  public <T> T exec(HttpVerb method, WebResource resource, GenericType<T> t)
+      throws IOException {
+    return restClient.exec(method, resource, t);
+  }
+
+  /**
+   * Execute the  GET operation. Failures are raised as IOException subclasses
+   * @param resource resource to work against
+   * @param c class to build
+   * @param <T> type expected
+   * @return an instance of the type T
+   * @throws IOException on any failure
+   */
+  public <T> T get(WebResource resource, Class<T> c) throws IOException {
+    return restClient.get(resource, c);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
new file mode 100644
index 0000000..b0eddb8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry.retrieve;
+
+import com.beust.jcommander.Strings;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.exceptions.RegistryIOException;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import static org.apache.slider.client.ClientRegistryBinder.*;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.ExceptionConverter;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Registry retriever. 
+ * This hides the HTTP operations that take place to
+ * get the actual content
+ */
+public class RegistryRetriever extends AMWebClient {
+  private static final Logger log = LoggerFactory.getLogger(RegistryRetriever.class);
+
+  private final String externalConfigurationURL;
+  private final String internalConfigurationURL;
+  private final String externalExportsURL;
+  private final String internalExportsURL;
+
+  /**
+   * Retrieve from a service by locating the
+   * exported {@link CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API}
+   * and working off it.
+   *
+   * @param conf configuration to work from
+   * @param record service record
+   * @throws RegistryIOException the address type of the endpoint does
+   * not match that expected (i.e. not a list of URLs), missing endpoint...
+   */
+  public RegistryRetriever(Configuration conf, ServiceRecord record) throws RegistryIOException {
+    super(conf);
+    externalConfigurationURL = lookupRestAPI(record,
+        PUBLISHER_CONFIGURATIONS_API, true);
+    internalConfigurationURL = lookupRestAPI(record,
+        PUBLISHER_CONFIGURATIONS_API, false);
+    externalExportsURL = lookupRestAPI(record,
+        PUBLISHER_EXPORTS_API, true);
+    internalExportsURL = lookupRestAPI(record,
+        PUBLISHER_EXPORTS_API, false);
+  }
+
+  /**
+   * Does a bonded registry retriever have a configuration?
+   * @param external flag to indicate that it is the external entries to fetch
+   * @return true if there is a URL to the configurations defined
+   */
+  public boolean hasConfigurations(boolean external) {
+    return !Strings.isStringEmpty(
+        external ? externalConfigurationURL : internalConfigurationURL);
+  }
+  
+  /**
+   * Get the configurations of the registry
+   * @param external flag to indicate that it is the external entries to fetch
+   * @return the configuration sets
+   */
+  public PublishedConfigSet getConfigurations(boolean external) throws
+      FileNotFoundException, IOException {
+
+    String confURL = getConfigurationURL(external);
+      WebResource webResource = resource(confURL);
+    return get(webResource, PublishedConfigSet.class);
+  }
+
+  protected String getConfigurationURL(boolean external) throws FileNotFoundException {
+    String confURL = external ? externalConfigurationURL: internalConfigurationURL;
+    if (Strings.isStringEmpty(confURL)) {
+      throw new FileNotFoundException("No configuration URL");
+    }
+    return confURL;
+  }
+
+  protected String getExportURL(boolean external) throws FileNotFoundException {
+    String confURL = external ? externalExportsURL: internalExportsURL;
+    if (Strings.isStringEmpty(confURL)) {
+      throw new FileNotFoundException("No configuration URL");
+    }
+    return confURL;
+  }
+
+  /**
+   * Get the configurations of the registry
+   * @param external flag to indicate that it is the external entries to fetch
+   * @return the configuration sets
+   */
+  public PublishedExportsSet getExports(boolean external) throws
+      FileNotFoundException, IOException {
+
+    String exportsUrl = getExportURL(external);
+    WebResource webResource = resource(exportsUrl);
+    return get(webResource, PublishedExportsSet.class);
+  }
+
+
+  /**
+   * Get a complete configuration, with all values
+   * @param configSet config set to ask for
+   * @param name name of the configuration
+   * @param external flag to indicate that it is an external configuration
+   * @return the retrieved config
+   * @throws IOException IO problems
+   */
+  public PublishedConfiguration retrieveConfiguration(PublishedConfigSet configSet,
+      String name,
+      boolean external) throws IOException {
+    String confURL = getConfigurationURL(external);
+    if (!configSet.contains(name)) {
+      throw new FileNotFoundException("Unknown configuration " + name);
+    }
+    confURL = SliderUtils.appendToURL(confURL, name);
+    WebResource webResource = resource(confURL);
+    return get(webResource, PublishedConfiguration.class);
+  }
+
+  /**
+   * Get a complete export, with all values
+   * @param exportSet
+   * @param name name of the configuration
+   * @param external flag to indicate that it is an external configuration
+   * @return the retrieved config
+   * @throws IOException IO problems
+   */
+  public PublishedExports retrieveExports(PublishedExportsSet exportSet,
+                                                      String name,
+                                                      boolean external) throws IOException {
+    if (!exportSet.contains(name)) {
+      throw new FileNotFoundException("Unknown export " + name);
+    }
+    String exportsURL = getExportURL(external);
+    exportsURL = SliderUtils.appendToURL(exportsURL, name);
+    return get(resource(exportsURL), PublishedExports.class);
+ }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("RegistryRetriever{");
+    sb.append("externalConfigurationURL='")
+      .append(externalConfigurationURL)
+      .append('\'');
+    sb.append(", internalConfigurationURL='")
+      .append(internalConfigurationURL)
+      .append('\'');
+    sb.append(", externalExportsURL='").append(externalExportsURL).append('\'');
+    sb.append(", internalExportsURL='").append(internalExportsURL).append('\'');
+    sb.append('}');
+    return sb.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
new file mode 100644
index 0000000..0266223
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A response for use as a return value from operations
+ */
+public class HttpOperationResponse {
+  
+  public int responseCode;
+  public long lastModified;
+  public String contentType;
+  public byte[] data;
+  public Map<String, List<String>> headers;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
new file mode 100644
index 0000000..c040345
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpVerb.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+/**
+ * Http verbs with details on what they support in terms of submit and
+ * response bodies.
+ * <p>
+ * Those verbs which do support bodies in the response MAY NOT return it;
+ * if the response code is 204 then the answer is "no body", but the operation
+ * is considered a success.
+ */
+public enum HttpVerb {
+  GET("GET", false, true),
+  POST("POST", true, true),
+  PUT("PUT", true, true),
+  DELETE("DELETE", false, true),
+  HEAD("HEAD", false, false);
+  
+  private final String verb;
+  private final boolean hasUploadBody;
+  private final boolean hasResponseBody;
+
+  HttpVerb(String verb, boolean hasUploadBody, boolean hasResponseBody) {
+    this.verb = verb;
+    this.hasUploadBody = hasUploadBody;
+    this.hasResponseBody = hasResponseBody;
+  }
+
+  public String getVerb() {
+    return verb;
+  }
+
+  public boolean hasUploadBody() {
+    return hasUploadBody;
+  }
+
+  public boolean hasResponseBody() {
+    return hasResponseBody;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82fdd408/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
new file mode 100644
index 0000000..e453f52
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/SliderURLConnectionFactory.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.web.KerberosUgiAuthenticator;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
+
+/**
+ * Factory for URL connections; used behind the scenes in the Jersey integration.
+ * <p>
+ * Derived from the WebHDFS implementation.
+ */
+public class SliderURLConnectionFactory {
+  private static final Logger log =
+      LoggerFactory.getLogger(SliderURLConnectionFactory.class);
+
+  /**
+   * Timeout for socket connects and reads
+   */
+  public final static int DEFAULT_SOCKET_TIMEOUT = 60 * 1000; // 1 minute
+  private final ConnectionConfigurator connConfigurator;
+
+  private static final ConnectionConfigurator DEFAULT_CONFIGURATOR = new BasicConfigurator();
+
+  /**
+   * Construct a new URLConnectionFactory based on the configuration. It will
+   * try to load SSL certificates when it is specified.
+   */
+  public static SliderURLConnectionFactory newInstance(Configuration conf) {
+    ConnectionConfigurator conn;
+    try {
+      conn = newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+    } catch (Exception e) {
+      log.debug("Cannot load customized SSL configuration.", e);
+      conn = DEFAULT_CONFIGURATOR;
+    }
+    return new SliderURLConnectionFactory(conn);
+  }
+
+  private SliderURLConnectionFactory(ConnectionConfigurator connConfigurator) {
+    this.connConfigurator = connConfigurator;
+  }
+
+  /**
+   * Create a new ConnectionConfigurator for SSL connections
+   */
+  private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+      Configuration conf) throws IOException, GeneralSecurityException {
+    final SSLFactory factory;
+    final SSLSocketFactory sf;
+    final HostnameVerifier hv;
+
+    factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+    factory.init();
+    sf = factory.createSSLSocketFactory();
+    hv = factory.getHostnameVerifier();
+
+    return new ConnectionConfigurator() {
+      @Override
+      public HttpURLConnection configure(HttpURLConnection conn)
+          throws IOException {
+        if (conn instanceof HttpsURLConnection) {
+          HttpsURLConnection c = (HttpsURLConnection) conn;
+          c.setSSLSocketFactory(sf);
+          c.setHostnameVerifier(hv);
+        }
+        SliderURLConnectionFactory.setupConnection(conn, timeout);
+        return conn;
+      }
+    };
+  }
+
+  /**
+   * Opens a url with read and connect timeouts
+   *
+   * @param url
+   *          to open
+   * @return URLConnection
+   * @throws IOException
+   */
+  public URLConnection openConnection(URL url) throws IOException {
+    try {
+      return openConnection(url, false);
+    } catch (AuthenticationException e) {
+      // Unreachable
+      return null;
+    }
+  }
+
+  /**
+   * Opens a url with read and connect timeouts
+   *
+   * @param url
+   *          URL to open
+   * @param isSpnego
+   *          whether the url should be authenticated via SPNEGO
+   * @return URLConnection
+   * @throws IOException
+   * @throws AuthenticationException
+   */
+  public URLConnection openConnection(URL url, boolean isSpnego)
+      throws IOException, AuthenticationException {
+    if (isSpnego) {
+        log.debug("open AuthenticatedURL connection {}", url);
+      UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
+      final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
+      return new AuthenticatedURL(new KerberosUgiAuthenticator(),
+          connConfigurator).openConnection(url, authToken);
+    } else {
+      log.debug("open URL connection {}", url);
+      URLConnection connection = url.openConnection();
+      if (connection instanceof HttpURLConnection) {
+        connConfigurator.configure((HttpURLConnection) connection);
+      }
+      return connection;
+    }
+  }
+
+  /**
+   * Sets connection parameters on the given URLConnection
+   * 
+   * @param connection
+   *          URLConnection to set
+   * @param socketTimeout
+   *          the connection and read timeout of the connection.
+   */
+  private static void setupConnection(URLConnection connection, int socketTimeout) {
+    connection.setConnectTimeout(socketTimeout);
+    connection.setReadTimeout(socketTimeout);
+    connection.setUseCaches(false);
+    if (connection instanceof HttpURLConnection) {
+      ((HttpURLConnection) connection).setInstanceFollowRedirects(true);
+    }
+  }
+
+  private static class BasicConfigurator implements ConnectionConfigurator {
+    @Override
+    public HttpURLConnection configure(HttpURLConnection conn)
+        throws IOException {
+      SliderURLConnectionFactory.setupConnection(conn, DEFAULT_SOCKET_TIMEOUT);
+      return conn;
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message