hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [03/39] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)
Date Tue, 02 Aug 2016 16:26:44 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
new file mode 100644
index 0000000..e436ae9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -0,0 +1,495 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import com.google.inject.Singleton;
+import org.apache.commons.io.FileUtils;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.text.MessageFormat;
+
+@Singleton
+public class CertificateManager {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(CertificateManager.class);
+
+  private static final String GEN_SRVR_KEY = "openssl genrsa -des3 " +
+      "-passout pass:{0} -out {1}" + File.separator + "{2} 4096 ";
+  private static final String GEN_SRVR_REQ = "openssl req -passin pass:{0} " +
+      "-new -key {1}" + File.separator + "{2} -out {1}" + File.separator +
+      "{5} -config {1}" + File.separator + "ca.config " +
+      "-subj {6} -batch";
+  private static final String SIGN_SRVR_CRT = "openssl ca -create_serial " +
+    "-out {1}" + File.separator + "{3} -days 365 -keyfile {1}" + File.separator
+    + "{2} -key {0} -selfsign -extensions jdk7_ca -config {1}" + File.separator
+    + "ca.config -batch -infiles {1}" + File.separator + "{5}";
+  private static final String EXPRT_KSTR = "openssl pkcs12 -export" +
+      " -in {2}" + File.separator + "{4} -inkey {2}" + File.separator +
+      "{3} -certfile {2}" + File.separator + "{4} -out {2}" + File.separator +
+      "{5} -password pass:{1} -passin pass:{0} \n";
+  private static final String REVOKE_AGENT_CRT = "openssl ca " +
+      "-config {0}" + File.separator + "ca.config -keyfile {0}" +
+      File.separator + "{4} -revoke {0}" + File.separator + "{2} -batch " +
+      "-passin pass:{3} -cert {0}" + File.separator + "{5}";
+  private static final String SIGN_AGENT_CRT = "openssl ca -config " +
+      "{0}" + File.separator + "ca.config -in {0}" + File.separator +
+      "{1} -out {0}" + File.separator + "{2} -batch -passin pass:{3} " +
+      "-keyfile {0}" + File.separator + "{4} -cert {0}" + File.separator + "{5}";
+  private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
+      "rsa:1024 -nodes -keyout {0}" + File.separator +
+      "{2}.key -subj {1} -out {0}" + File.separator + "{2}.csr " +
+      "-config {3}" + File.separator + "ca.config ";
+  private String passphrase;
+  private String applicationName;
+
+
+  public void initialize(MapOperations compOperations) throws SliderException {
+    String hostname = null;
+    try {
+      hostname = InetAddress.getLocalHost().getCanonicalHostName();
+    } catch (UnknownHostException e) {
+      hostname = "localhost";
+    }
+    this.initialize(compOperations, hostname, null, null);
+  }
+
+  /**
+    * Verify that root certificate exists, generate it otherwise.
+    */
+  public void initialize(MapOperations compOperations,
+                         String hostname, String containerId,
+                         String appName) throws SliderException {
+    SecurityUtils.initializeSecurityParameters(compOperations);
+
+    LOG.info("Initialization of root certificate");
+    boolean certExists = isCertExists();
+    LOG.info("Certificate exists:" + certExists);
+
+    this.applicationName = appName;
+
+    if (!certExists) {
+      generateAMKeystore(hostname, containerId);
+    }
+
+  }
+
+  /**
+   * Checks root certificate state.
+   * @return "true" if certificate exists
+   */
+  private boolean isCertExists() {
+
+    String srvrKstrDir = SecurityUtils.getSecurityDir();
+    String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+    File certFile = new File(srvrKstrDir + File.separator + srvrCrtName);
+    LOG.debug("srvrKstrDir = " + srvrKstrDir);
+    LOG.debug("srvrCrtName = " + srvrCrtName);
+    LOG.debug("certFile = " + certFile.getAbsolutePath());
+
+    return certFile.exists();
+  }
+
+  public void setPassphrase(String passphrase) {
+    this.passphrase = passphrase;
+  }
+
+  class StreamConsumer extends Thread
+  {
+    InputStream is;
+    boolean logOutput;
+
+    StreamConsumer(InputStream is, boolean logOutput)
+    {
+      this.is = is;
+      this.logOutput = logOutput;
+    }
+
+    StreamConsumer(InputStream is)
+    {
+      this(is, false);
+    }
+
+    public void run()
+    {
+      try
+      {
+        InputStreamReader isr = new InputStreamReader(is,
+                                                      Charset.forName("UTF8"));
+        BufferedReader br = new BufferedReader(isr);
+        String line;
+        while ( (line = br.readLine()) != null)
+          if (logOutput) {
+            LOG.info(line);
+          }
+      } catch (IOException e)
+      {
+        LOG.error("Error during processing of process stream", e);
+      }
+    }
+  }
+
+
+  /**
+   * Runs os command
+   *
+   * @return command execution exit code
+   */
+  private int runCommand(String command) throws SliderException {
+    int exitCode = -1;
+    String line = null;
+    Process process = null;
+    BufferedReader br= null;
+    try {
+      process = Runtime.getRuntime().exec(command);
+      StreamConsumer outputConsumer =
+          new StreamConsumer(process.getInputStream(), true);
+      StreamConsumer errorConsumer =
+          new StreamConsumer(process.getErrorStream(), true);
+
+      outputConsumer.start();
+      errorConsumer.start();
+
+      try {
+        process.waitFor();
+        SecurityUtils.logOpenSslExitCode(command, process.exitValue());
+        exitCode = process.exitValue();
+        if (exitCode != 0) {
+          throw new SliderException(exitCode, "Error running command %s", command);
+        }
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    } catch (IOException e) {
+      e.printStackTrace();
+    } finally {
+      if (br != null) {
+        try {
+          br.close();
+        } catch (IOException ioe) {
+          ioe.printStackTrace();
+        }
+      }
+    }
+
+    return exitCode;//some exception occurred
+
+  }
+
+  public synchronized void generateContainerCertificate(String hostname,
+                                                        String identifier) {
+    LOG.info("Generation of certificate for {}", hostname);
+
+    String srvrKstrDir = SecurityUtils.getSecurityDir();
+    Object[] scriptArgs = {srvrKstrDir, getSubjectDN(hostname, identifier,
+        this.applicationName), identifier, SecurityUtils.getSecurityDir()};
+
+    try {
+      String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs);
+      runCommand(command);
+
+      signAgentCertificate(identifier);
+
+    } catch (SliderException e) {
+      LOG.error("Error generating the agent certificate", e);
+    }
+  }
+
+  public synchronized SecurityStore generateContainerKeystore(String hostname,
+                                                              String requesterId,
+                                                              String role,
+                                                              String keystorePass)
+      throws SliderException {
+    LOG.info("Generation of container keystore for container {} on {}",
+             requesterId, hostname);
+
+    generateContainerCertificate(hostname, requesterId);
+
+    // come up with correct args to invoke keystore command
+    String srvrCrtPass = SecurityUtils.getKeystorePass();
+    String srvrKstrDir = SecurityUtils.getSecurityDir();
+    String containerCrtName = requesterId + ".crt";
+    String containerKeyName = requesterId + ".key";
+    String kstrName = getKeystoreFileName(requesterId, role);
+
+    Object[] scriptArgs = {srvrCrtPass, keystorePass, srvrKstrDir,
+        containerKeyName, containerCrtName, kstrName};
+
+    String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
+    runCommand(command);
+
+    return new SecurityStore(new File(srvrKstrDir, kstrName),
+                             SecurityStore.StoreType.keystore);
+  }
+
+  private static String getKeystoreFileName(String containerId,
+                                            String role) {
+    return String.format("keystore-%s-%s.p12", containerId,
+                         role != null ? role : "");
+  }
+
+  private void generateAMKeystore(String hostname, String containerId)
+      throws SliderException {
+    LOG.info("Generation of server certificate");
+
+    String srvrKstrDir = SecurityUtils.getSecurityDir();
+    String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+    String srvrCsrName = SliderKeys.CSR_FILE_NAME;
+    String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+    String kstrName = SliderKeys.KEYSTORE_FILE_NAME;
+    String srvrCrtPass = SecurityUtils.getKeystorePass();
+
+    Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName,
+        srvrCrtName, kstrName, srvrCsrName, getSubjectDN(hostname, containerId,
+        this.applicationName)};
+
+    String command = MessageFormat.format(GEN_SRVR_KEY, scriptArgs);
+    runCommand(command);
+
+    command = MessageFormat.format(GEN_SRVR_REQ, scriptArgs);
+    runCommand(command);
+
+    command = MessageFormat.format(SIGN_SRVR_CRT, scriptArgs);
+    runCommand(command);
+
+    Object[] keystoreArgs = {srvrCrtPass, srvrCrtPass, srvrKstrDir, srvrKeyName,
+        srvrCrtName, kstrName, srvrCsrName};
+    command = MessageFormat.format(EXPRT_KSTR, keystoreArgs);
+    runCommand(command);
+  }
+
+  public SecurityStore generateContainerTruststore(String containerId,
+                                                   String role,
+                                                   String truststorePass)
+      throws SliderException {
+
+    String srvrKstrDir = SecurityUtils.getSecurityDir();
+    String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+    String srvrCsrName = SliderKeys.CSR_FILE_NAME;
+    String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+    String kstrName = getTruststoreFileName(role, containerId);
+    String srvrCrtPass = SecurityUtils.getKeystorePass();
+
+    Object[] scriptArgs = {srvrCrtPass, truststorePass, srvrKstrDir, srvrKeyName,
+        srvrCrtName, kstrName, srvrCsrName};
+
+    String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
+    runCommand(command);
+
+    return new SecurityStore(new File(srvrKstrDir, kstrName),
+                             SecurityStore.StoreType.truststore);
+  }
+
+  private static String getTruststoreFileName(String role, String containerId) {
+    return String.format("truststore-%s-%s.p12", containerId,
+                         role != null ? role : "");
+  }
+
+  /**
+   * Returns server certificate content
+   * @return string with server certificate content
+   */
+  public String getServerCert() {
+    File certFile = getServerCertficateFilePath();
+    String srvrCrtContent = null;
+    try {
+      srvrCrtContent = FileUtils.readFileToString(certFile);
+    } catch (IOException e) {
+      LOG.error(e.getMessage());
+    }
+    return srvrCrtContent;
+  }
+
+  public static File getServerCertficateFilePath() {
+    return new File(String.format("%s%s%s",
+                                  SecurityUtils.getSecurityDir(),
+                                  File.separator,
+                                  SliderKeys.CRT_FILE_NAME));
+  }
+
+  public static File getAgentCertficateFilePath(String containerId) {
+    return new File(String.format("%s%s%s.crt",
+                                  SecurityUtils.getSecurityDir(),
+                                  File.separator,
+                                  containerId));
+  }
+
+  public static File getContainerKeystoreFilePath(String containerId,
+                                                  String role) {
+    return new File(SecurityUtils.getSecurityDir(), getKeystoreFileName(
+        containerId,
+        role
+    ));
+  }
+
+  public static File getContainerTruststoreFilePath(String role,
+                                                    String containerId) {
+    return new File(SecurityUtils.getSecurityDir(),
+                    getTruststoreFileName(role, containerId));
+  }
+
+  public static File getAgentKeyFilePath(String containerId) {
+    return new File(String.format("%s%s%s.key",
+                                  SecurityUtils.getSecurityDir(),
+                                  File.separator,
+                                  containerId));
+  }
+
+  /**
+   * Signs agent certificate
+   * Adds agent certificate to server keystore
+   * @return string with agent signed certificate content
+   */
+  public synchronized SignCertResponse signAgentCrt(String agentHostname,
+                                                    String agentCrtReqContent,
+                                                    String passphraseAgent) {
+    SignCertResponse response = new SignCertResponse();
+    LOG.info("Signing of agent certificate");
+    LOG.info("Verifying passphrase");
+
+    if (!this.passphrase.equals(passphraseAgent.trim())) {
+      LOG.warn("Incorrect passphrase from the agent");
+      response.setResult(SignCertResponse.ERROR_STATUS);
+      response.setMessage("Incorrect passphrase from the agent");
+      return response;
+    }
+
+    String srvrKstrDir = SecurityUtils.getSecurityDir();
+    String srvrCrtPass = SecurityUtils.getKeystorePass();
+    String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+    String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+    String agentCrtReqName = agentHostname + ".csr";
+    String agentCrtName = agentHostname + ".crt";
+
+    Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
+        srvrCrtPass, srvrKeyName, srvrCrtName};
+
+    //Revoke previous agent certificate if exists
+    File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+
+    String command = null;
+    if (agentCrtFile.exists()) {
+      LOG.info("Revoking of " + agentHostname + " certificate.");
+      command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+      try {
+        runCommand(command);
+      } catch (SliderException e) {
+        int commandExitCode = e.getExitCode();
+        response.setResult(SignCertResponse.ERROR_STATUS);
+        response.setMessage(
+            SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
+        return response;
+      }
+    }
+
+    File agentCrtReqFile = new File(srvrKstrDir + File.separator +
+        agentCrtReqName);
+    try {
+      FileUtils.writeStringToFile(agentCrtReqFile, agentCrtReqContent);
+    } catch (IOException e1) {
+      // TODO Auto-generated catch block
+      e1.printStackTrace();
+    }
+
+    command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
+
+    LOG.debug(SecurityUtils.hideOpenSslPassword(command));
+    try {
+      runCommand(command);
+    } catch (SliderException e) {
+      int commandExitCode = e.getExitCode();
+      response.setResult(SignCertResponse.ERROR_STATUS);
+      response.setMessage(
+          SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
+      return response;
+    }
+
+    String agentCrtContent = "";
+    try {
+      agentCrtContent = FileUtils.readFileToString(agentCrtFile);
+    } catch (IOException e) {
+      e.printStackTrace();
+      LOG.error("Error reading signed agent certificate");
+      response.setResult(SignCertResponse.ERROR_STATUS);
+      response.setMessage("Error reading signed agent certificate");
+      return response;
+    }
+    response.setResult(SignCertResponse.OK_STATUS);
+    response.setSignedCa(agentCrtContent);
+    //LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode));
+    return response;
+  }
+
+  private String signAgentCertificate (String containerId)
+      throws SliderException {
+    String srvrKstrDir = SecurityUtils.getSecurityDir();
+    String srvrCrtPass = SecurityUtils.getKeystorePass();
+    String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+    String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+    String agentCrtReqName = containerId + ".csr";
+    String agentCrtName = containerId + ".crt";
+
+    // server certificate must exist already
+    if (!(new File(srvrKstrDir, srvrCrtName).exists())) {
+      throw new SliderException("CA certificate not generated");
+    }
+
+    Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
+        srvrCrtPass, srvrKeyName, srvrCrtName};
+
+    //Revoke previous agent certificate if exists
+    File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+
+    String command;
+    if (agentCrtFile.exists()) {
+      LOG.info("Revoking of " + containerId + " certificate.");
+      command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+      runCommand(command);
+    }
+
+    command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
+
+    LOG.debug(SecurityUtils.hideOpenSslPassword(command));
+    runCommand(command);
+
+    return agentCrtName;
+
+  }
+
+  private String getSubjectDN(String hostname, String containerId,
+                              String appName) {
+    return String.format("/CN=%s%s%s",
+                         hostname,
+                         containerId != null ? "/OU=" + containerId : "",
+                         appName != null ? "/OU=" + appName : "");
+
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
new file mode 100644
index 0000000..e2339d5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ *
+ */
+public class KeystoreGenerator extends AbstractSecurityStoreGenerator {
+
+
+  public KeystoreGenerator(CertificateManager certificateMgr) {
+    super(certificateMgr);
+  }
+
+  @Override
+  public SecurityStore generate(String hostname, String containerId,
+                                AggregateConf instanceDefinition,
+                                MapOperations compOps, String role)
+      throws SliderException, IOException {
+    SecurityStore keystore = null;
+    String password = getStorePassword(
+        instanceDefinition.getAppConf().credentials, compOps, role);
+    if (password != null) {
+      keystore =
+          certificateMgr.generateContainerKeystore(hostname, containerId, role,
+                                                   password);
+    }
+    return keystore;
+  }
+
+  @Override
+  String getPassword(MapOperations compOps) {
+    return compOps.get(
+        compOps.get(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY));
+  }
+
+  @Override
+  String getAlias(MapOperations compOps) {
+    return compOps.getOption(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
+                             SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
new file mode 100644
index 0000000..fc54267
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import java.io.File;
+
+/**
+ *
+ */
+public class SecurityStore {
+  private File file;
+
+  public enum StoreType {truststore, keystore}
+
+  private StoreType type;
+
+  public String getType() {
+    return type.name();
+  }
+
+  public File getFile() {
+    return file;
+  }
+
+  public SecurityStore(File file,
+                       StoreType type) {
+
+    this.file = file;
+    this.type = type;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    SecurityStore that = (SecurityStore) o;
+
+    if (file != null ? !file.equals(that.file) : that.file != null)
+      return false;
+    if (type != that.type) return false;
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = file != null ? file.hashCode() : 0;
+    result = 31 * result + (type != null ? type.hashCode() : 0);
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
new file mode 100644
index 0000000..a814988
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ *
+ */
+public interface SecurityStoreGenerator {
+
+  SecurityStore generate(String hostname,
+                         String containerId,
+                         AggregateConf instanceDefinition,
+                         MapOperations compOps,
+                         String role)
+      throws SliderException, IOException;
+
+  boolean isStoreRequested(MapOperations compOps);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
new file mode 100644
index 0000000..e82ad84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+//import java.nio.file.Files;
+//import java.nio.file.Path;
+//import java.nio.file.Paths;
+//import java.nio.file.attribute.PosixFilePermission;
+//import java.nio.file.attribute.PosixFilePermissions;
+
+
+/**
+ *
+ */
+public class SecurityUtils {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SecurityUtils.class);
+
+  private static String CA_CONFIG_CONTENTS =  "HOME            = .\n"
+                                            + "RANDFILE        = $ENV::HOME/.rnd\n\n"
+                                            + "[ ca ]\n"
+                                            + "default_ca             = CA_CLIENT\n"
+                                            + "[ CA_CLIENT ]\n"
+                                            + "dir                    = ${SEC_DIR}/db\n"
+                                            + "certs                  = $dir/certs\n"
+                                            + "new_certs_dir          = $dir/newcerts\n"
+                                            + "\n"
+                                            + "database               = $dir/index.txt\n"
+                                            + "serial                 = $dir/serial\n"
+                                            + "default_days           = 365    \n"
+                                            + "\n"
+                                            + "default_crl_days       = 7  \n"
+                                            + "default_md             = sha256 \n"
+                                            + "\n"
+                                            + "policy                 = policy_anything \n"
+                                            + "\n"
+                                            + "[ policy_anything ]\n"
+                                            + "countryName            = optional\n"
+                                            + "stateOrProvinceName    = optional\n"
+                                            + "localityName           = optional\n"
+                                            + "organizationName       = optional\n"
+                                            + "organizationalUnitName = optional\n"
+                                            + "commonName             = optional\n"
+                                            + "emailAddress           = optional\n"
+                                            + "\n"
+                                            + "[req]\n"
+                                            + "distinguished_name     = req_distinguished_name\n"
+                                            + "\n"
+                                            + "[ req_distinguished_name ]\n"
+                                            + "\n"
+                                            + "[ jdk7_ca ]\n"
+                                            + "subjectKeyIdentifier = hash\n"
+                                            + "authorityKeyIdentifier = keyid:always,issuer:always\n"
+                                            + "basicConstraints = CA:true\n";
+
+  private static final String PASS_TOKEN = "pass:";
+  private static String keystorePass;
+  private static String securityDir;
+
+  public static void logOpenSslExitCode(String command, int exitCode) {
+    if (exitCode == 0) {
+      LOG.info(getOpenSslCommandResult(command, exitCode));
+    } else {
+      LOG.warn(getOpenSslCommandResult(command, exitCode));
+    }
+
+  }
+
+  public static String hideOpenSslPassword(String command){
+    int start = command.indexOf(PASS_TOKEN);
+    while (start >= 0) {
+      start += PASS_TOKEN.length();
+      CharSequence cs = command.subSequence(start, command.indexOf(" ", start));
+      command = command.replace(cs, "****");
+      start = command.indexOf(PASS_TOKEN, start + 1);
+    }
+    return command;
+  }
+
+  public static String getOpenSslCommandResult(String command, int exitCode) {
+    return new StringBuilder().append("Command ")
+        .append(hideOpenSslPassword(command))
+        .append(" was finished with exit code: ")
+        .append(exitCode).append(" - ")
+        .append(getOpenSslExitCodeDescription(exitCode)).toString();
+  }
+
+  private static String getOpenSslExitCodeDescription(int exitCode) {
+    switch (exitCode) {
+      case 0: {
+        return "the operation was completed successfully.";
+      }
+      case 1: {
+        return "an error occurred parsing the command options.";
+      }
+      case 2: {
+        return "one of the input files could not be read.";
+      }
+      case 3: {
+        return "an error occurred creating the PKCS#7 file or when reading the MIME message.";
+      }
+      case 4: {
+        return "an error occurred decrypting or verifying the message.";
+      }
+      case 5: {
+        return "the message was verified correctly but an error occurred writing out the signers certificates.";
+      }
+      default:
+        return "unsupported code";
+    }
+  }
+
+  public static void writeCaConfigFile(String path) throws IOException {
+    String contents = CA_CONFIG_CONTENTS.replace("${SEC_DIR}", path);
+    FileUtils.writeStringToFile(new File(path, "ca.config"), contents);
+  }
+
+  public static String getKeystorePass() {
+    return keystorePass;
+  }
+
+  public static String getSecurityDir() {
+    return securityDir;
+  }
+
+  public static void    initializeSecurityParameters(MapOperations configMap) {
+    initializeSecurityParameters(configMap, false);
+  }
+
+  public static void initializeSecurityParameters(MapOperations configMap,
+                                                boolean persistPassword) {
+    String keyStoreLocation = configMap.getOption(
+        SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation());
+    if (keyStoreLocation == null) {
+      LOG.error(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION
+          + " is not specified. Unable to initialize security params.");
+      return;
+    }
+    File secDirFile = new File(keyStoreLocation).getParentFile();
+    if (!secDirFile.exists()) {
+      // create entire required directory structure
+      File dbDir = new File(secDirFile, "db");
+      File newCertsDir = new File(dbDir, "newcerts");
+      newCertsDir.mkdirs();
+      RawLocalFileSystem fileSystem = null;
+      try {
+        fileSystem = new RawLocalFileSystem();
+        FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE,
+                                                    FsAction.NONE);
+        fileSystem.setPermission(new Path(dbDir.getAbsolutePath()),
+                                 permissions);
+        fileSystem.setPermission(new Path(dbDir.getAbsolutePath()), permissions);
+        fileSystem.setPermission(new Path(newCertsDir.getAbsolutePath()),
+                                 permissions);
+        File indexFile = new File(dbDir, "index.txt");
+        indexFile.createNewFile();
+        SecurityUtils.writeCaConfigFile(secDirFile.getAbsolutePath().replace('\\', '/'));
+
+      } catch (IOException e) {
+        LOG.error("Unable to create SSL configuration directories/files", e);
+      } finally {
+        if (fileSystem != null) {
+          try {
+            fileSystem.close();
+          } catch (IOException e) {
+            LOG.warn("Unable to close fileSystem", e);
+          }
+        }
+      }
+      // need to create the password
+    }
+    keystorePass = getKeystorePassword(secDirFile, persistPassword);
+    securityDir = secDirFile.getAbsolutePath();
+  }
+
+  private static String getKeystorePassword(File secDirFile,
+                                            boolean persistPassword) {
+    File passFile = new File(secDirFile, SliderKeys.CRT_PASS_FILE_NAME);
+    String password = null;
+    if (!passFile.exists()) {
+      LOG.info("Generating keystore password");
+      password = RandomStringUtils.randomAlphanumeric(
+          Integer.valueOf(SliderKeys.PASS_LEN));
+      if (persistPassword) {
+        try {
+          FileUtils.writeStringToFile(passFile, password);
+          passFile.setWritable(true);
+          passFile.setReadable(true);
+        } catch (IOException e) {
+          e.printStackTrace();
+          throw new RuntimeException(
+              "Error creating certificate password file");
+        }
+      }
+    } else {
+      LOG.info("Reading password from existing file");
+      try {
+        password = FileUtils.readFileToString(passFile);
+        password = password.replaceAll("\\p{Cntrl}", "");
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+    }
+
+    return password;
+  }
+
+  private static String getDefaultKeystoreLocation() {
+    File workDir = null;
+    try {
+      workDir =  new File(FileUtils.getTempDirectory().getAbsolutePath()
+                          + "/sec" + System.currentTimeMillis());
+      if (!workDir.mkdirs()) {
+        throw new IOException("Unable to create temporary security directory");
+      }
+    } catch (IOException e) {
+      LOG.warn("Unable to create security directory");
+      return null;
+    }
+
+    return new StringBuilder().append(workDir.getAbsolutePath())
+        .append(File.separator)
+        .append(SliderKeys.SECURITY_DIR)
+        .append(File.separator)
+        .append(SliderKeys.KEYSTORE_FILE_NAME).toString();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
new file mode 100644
index 0000000..8437d88
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.security;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ *
+ * Sign certificate response data model.
+ *
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name = "", propOrder = {})
+public class SignCertResponse {
+	
+  public static final String ERROR_STATUS = "ERROR";
+  public static final String OK_STATUS = "OK";
+
+  @XmlElement
+  private String result;
+  @XmlElement
+  private String signedCa;
+  @XmlElement
+  private String message;
+
+  public String getResult() {
+    return result;
+  }
+  public void setResult(String result) {
+    this.result = result;
+  }
+  public String getSignedCa() {
+    return signedCa;
+  }
+  public void setSignedCa(String signedCa) {
+    this.signedCa = signedCa;
+  }
+
+  public String getMessage() {
+    return message;
+  }
+  public void setMessage(String message) {
+    this.message = message;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
new file mode 100644
index 0000000..4bccb87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.security;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ *
+ * Sign certificate request data model.
+ *
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name = "", propOrder = {})
+public class SignMessage {
+
+  @XmlElement
+  private String csr;
+  @XmlElement
+  private String passphrase;
+  public String getCsr() {
+    return csr;
+  }
+  public void setCsr(String csr) {
+    this.csr = csr;
+  }
+  public String getPassphrase() {
+    return passphrase;
+  }
+  public void setPassphrase(String passphrase) {
+    this.passphrase = passphrase;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
new file mode 100644
index 0000000..226250f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class StoresGenerator {
+
+  static CertificateManager certMgr = new CertificateManager();
+  private static SecurityStoreGenerator[] GENERATORS = {
+      new KeystoreGenerator(certMgr), new TruststoreGenerator(certMgr)
+  };
+
+  public static SecurityStore[] generateSecurityStores(String hostname,
+                                                       String containerId,
+                                                       String role,
+                                                       AggregateConf instanceDefinition,
+                                                       MapOperations compOps)
+      throws SliderException, IOException {
+    //discover which stores need generation based on the passwords configured
+    List<SecurityStore> files = new ArrayList<SecurityStore>();
+    for (SecurityStoreGenerator generator : GENERATORS) {
+      if (generator.isStoreRequested(compOps)) {
+        SecurityStore store = generator.generate(hostname,
+                                                 containerId,
+                                                 instanceDefinition,
+                                                 compOps,
+                                                 role);
+        if (store != null) {
+          files.add(store);
+        }
+      }
+    }
+
+    if (files.isEmpty()) {
+      throw new SliderException("Security stores were requested but none were "
+                                + "generated. Check the AM logs and ensure "
+                                + "passwords are configured for the components "
+                                + "requiring the stores.");
+    }
+    return files.toArray(new SecurityStore[files.size()]);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
new file mode 100644
index 0000000..d16dcbd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class TruststoreGenerator extends AbstractSecurityStoreGenerator {
+
+
+  public TruststoreGenerator(CertificateManager certificateMgr) {
+    super(certificateMgr);
+  }
+
+  @Override
+  public SecurityStore generate(String hostname, String containerId,
+                                AggregateConf instanceDefinition,
+                                MapOperations compOps, String role)
+      throws SliderException, IOException {
+    SecurityStore truststore = null;
+    String password = getStorePassword(
+        instanceDefinition.getAppConf().credentials, compOps, role);
+    if (password != null) {
+      truststore = certificateMgr.generateContainerTruststore(containerId,
+                                                              role, password);
+    }
+    return truststore;
+  }
+
+  @Override
+  String getPassword(MapOperations compOps) {
+    return compOps.get(
+        compOps.get(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY));
+  }
+
+  @Override
+  String getAlias(MapOperations compOps) {
+    return compOps.getOption(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY,
+                             SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
new file mode 100644
index 0000000..1622309
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.zk.ZookeeperUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base service for the standard slider client/server services
+ */
+public abstract class AbstractSliderLaunchedService extends
+    LaunchedWorkflowCompositeService {
+  private static final Logger log =
+    LoggerFactory.getLogger(AbstractSliderLaunchedService.class);
+
+  protected AbstractSliderLaunchedService(String name) {
+    super(name);
+    // make sure all the yarn configs get loaded
+    YarnConfiguration conf = new YarnConfiguration();
+    ConfigHelper.registerDeprecatedConfigItems();
+  }
+
+  /**
+   * look up the registry quorum from the config
+   * @return the quorum string
+   * @throws BadConfigException if it is not there or invalid
+   */
+  public String lookupZKQuorum() throws BadConfigException {
+ 
+    String registryQuorum = getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+    
+    // though if neither is set: trouble
+    if (SliderUtils.isUnset(registryQuorum)) {
+      throw new BadConfigException(
+          "No Zookeeper quorum provided in the"
+          + " configuration property " + RegistryConstants.KEY_REGISTRY_ZK_QUORUM
+      );
+    }
+    ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
+    return registryQuorum;
+  }
+
+  /**
+   * Create, adopt ,and start the YARN registration service
+   * @return the registry operations service, already deployed as a child
+   * of the AbstractSliderLaunchedService instance.
+   */
+  public RegistryOperations startRegistryOperationsService()
+      throws BadConfigException {
+
+    // push back the slider registry entry if needed
+    String quorum = lookupZKQuorum();
+    RegistryOperations registryWriterService =
+        createRegistryOperationsInstance();
+    deployChildService(registryWriterService);
+    return registryWriterService;
+  }
+
+  /**
+   * Create the registry operations instance. This is to allow
+   * subclasses to instantiate a subclass service
+   * @return an instance to match to the lifecycle of this service
+   */
+  protected RegistryOperations createRegistryOperationsInstance() {
+    return RegistryOperationsFactory.createInstance("YarnRegistry", getConfig());
+  }
+
+  /**
+   * Utility method to require an argument to be set (non null, non-empty)
+   * @param argname argument name
+   * @param value value
+   * @throws BadCommandArgumentsException if the condition is not met
+   */
+  protected static void requireArgumentSet(String argname, String value)
+      throws BadCommandArgumentsException {
+    require(isSet(value), "Required argument %s missing", argname );
+  }
+
+  /**
+   * Require a condition to hold; throw {@link BadCommandArgumentsException} if not.
+   * The exception text is the formatted message.
+   * @param condition condition
+   * @param message string to format
+   * @param args list of arguments to format.
+   * @throws BadCommandArgumentsException
+   */
+  protected static void require(boolean condition, String message,
+      Object... args)
+      throws BadCommandArgumentsException {
+    if (!condition) {
+      throw new BadCommandArgumentsException(message, args);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
new file mode 100644
index 0000000..40ceab8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Wait for a service to stop.
+ * 
+ * WARNING: the notification may come in as soon as the service enters
+ * the stopped state: it may take some time for the actual stop operation
+ * to complete.
+ */
+public class EndOfServiceWaiter implements ServiceStateChangeListener {
+
+  private final AtomicBoolean finished = new AtomicBoolean(false);
+  private final String name;
+  private Service service;
+
+  /**
+   * Wait for a service; use the service name as this instance's name
+   * @param service service
+   */
+  public EndOfServiceWaiter(Service service) {
+    this(service.getName(), service);
+  }
+
+
+  /**
+   * Wait for a service
+   * @param name name for messages
+   * @param service service
+   */
+  public EndOfServiceWaiter(String name, Service service) {
+    this.name = name;
+    this.service = service;
+    service.registerServiceListener(this);
+  }
+
+  public synchronized void waitForServiceToStop(long timeout) throws
+      InterruptedException, TimeoutException {
+    service.waitForServiceToStop(timeout);
+    if (!finished.get()) {
+      wait(timeout);
+      if (!finished.get()) {
+        throw new TimeoutException(name
+                                   + " did not finish after " + timeout +
+                                   " milliseconds");
+      }
+    }
+  }
+
+  /**
+   * Wait for service state change callbacks; notify self if the service has
+   * now stopped
+   * @param service service
+   */
+  @Override
+  public synchronized void stateChanged(Service service) {
+    if (service.isInState(Service.STATE.STOPPED)) {
+      finished.set(true);
+      notify();
+    }
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
new file mode 100644
index 0000000..bcd1969
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.core.main.RunService;
+import org.apache.slider.server.services.workflow.WorkflowCompositeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a workflow compositoe service which can be launched from the CLI
+ * ... catches the arguments and implements a stub runService operation.
+ */
+public class LaunchedWorkflowCompositeService extends WorkflowCompositeService
+    implements RunService {
+  private static final Logger log = LoggerFactory.getLogger(
+      LaunchedWorkflowCompositeService.class);
+  private String[] argv;
+  
+  public LaunchedWorkflowCompositeService(String name) {
+    super(name);
+  }
+
+  public LaunchedWorkflowCompositeService(String name, Service... children) {
+    super(name, children);
+  }
+
+  /**
+   * Implementation of set-ness, groovy definition of true/false for a string
+   * @param s
+   * @return true iff the string is non-null and non-empty
+   */
+  protected static boolean isUnset(String s) {
+    return StringUtils.isEmpty(s);
+  }
+
+  protected static boolean isSet(String s) {
+    return StringUtils.isNotEmpty(s);
+  }
+
+  protected String[] getArgv() {
+    return argv;
+  }
+
+  /**
+   * Pre-init argument binding
+   * @param config the initial configuration build up by the
+   * service launcher.
+   * @param args argument list list of arguments passed to the command line
+   * after any launcher-specific commands have been stripped.
+   * @return the configuration
+   * @throws Exception
+   */
+  @Override
+  public Configuration bindArgs(Configuration config, String... args) throws
+                                                                      Exception {
+    this.argv = args;
+    if (log.isDebugEnabled()) {
+      log.debug("Binding {} Arguments:", args.length);
+
+      StringBuilder builder = new StringBuilder();
+      for (String arg : args) {
+        builder.append('"').append(arg).append("\" ");
+      }
+      log.debug(builder.toString());
+    }
+    return config;
+  }
+
+  @Override
+  public int runService() throws Throwable {
+    return LauncherExitCodes.EXIT_SUCCESS;
+  }
+
+  @Override
+  public synchronized void addService(Service service) {
+    Preconditions.checkArgument(service != null, "null service argument");
+    super.addService(service);
+  }
+
+  /**
+   * Run a child service -initing and starting it if this
+   * service has already passed those parts of its own lifecycle
+   * @param service the service to start
+   */
+  protected boolean deployChildService(Service service) {
+    service.init(getConfig());
+    addService(service);
+    if (isInState(STATE.STARTED)) {
+      service.start();
+      return true;
+    }
+    return false;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
new file mode 100644
index 0000000..6ab9de6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import java.util.regex.Pattern;
+
+/**
+ * Utility class to validate strings against a predefined pattern.
+ */
+public class PatternValidator {
+
+  public static final String E_INVALID_NAME =
+      "Invalid name %s does not match the pattern pattern %s ";
+  private final Pattern valid;
+  private final String pattern;
+
+  public PatternValidator(String pattern) {
+    this.pattern = pattern;
+    valid = Pattern.compile(pattern);
+  }
+
+  /**
+   * Validate the name -restricting it to the set defined in 
+   * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+   * @param name name to validate
+   * @throws IllegalArgumentException if not a valid name
+   */
+  public void validate(String name) {
+    if (!matches(name)) {
+      throw new IllegalArgumentException(
+          String.format(E_INVALID_NAME, name, pattern));
+    }
+  }
+
+  /**
+   * Query to see if the pattern matches
+   * @param name name to validate
+   * @return true if the string matches the pattern
+   */
+  public boolean matches(String name) {
+    return valid.matcher(name).matches();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
new file mode 100644
index 0000000..ebfcb99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.webapp.WebApp;
+
+/**
+ * Contains a webapp reference and stops it in teardown if non-null
+ * <p>
+ * It does not start the application.
+ * Access to the field is not synchronized across threads; it is the
+ * responsibility of the caller.
+ */
+public class WebAppService<T extends WebApp> extends AbstractService {
+
+  private volatile T webApp;
+
+  public WebAppService(String name) {
+    super(name);
+  }
+
+  public WebAppService(String name, T app) {
+    super(name);
+    webApp = app;
+  }
+
+  public T getWebApp() {
+    return webApp;
+  }
+
+  public void setWebApp(T webApp) {
+    this.webApp = webApp;
+  }
+
+
+  @Override
+  protected void serviceStart() throws Exception {
+
+  }
+
+  /**
+   * Stop operation stops the webapp; sets the reference to null
+   * @throws Exception
+   */
+  @Override
+  protected void serviceStop() throws Exception {
+    if (webApp != null) {
+      webApp.stop();
+      webApp = null;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
new file mode 100644
index 0000000..8b711aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.service.AbstractService;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Service that closes the closeable supplied during shutdown, if not null.
+ * 
+ * As the Service interface itself extends Closeable, this service
+ * can be used to shut down other services if desired.
+ */
+public class ClosingService<C extends Closeable> extends AbstractService {
+
+  private C closeable;
+
+  public ClosingService(String name) {
+    super(name);
+  }
+
+  /**
+   * Construct an instance of the service
+   * @param name service name
+   * @param closeable closeable to close (may be null)
+   */
+  public ClosingService(String name,
+      C closeable) {
+    super(name);
+    this.closeable = closeable;
+  }
+
+  /**
+   * Construct an instance of the service, using the default name
+   * @param closeable closeable to close (may be null)
+   */
+  public ClosingService(C closeable) {
+    this("ClosingService", closeable);
+  }
+
+
+  /**
+   * Get the closeable
+   * @return the closeable
+   */
+  public synchronized C getCloseable() {
+    return closeable;
+  }
+
+  /**
+   * Set or update the closeable.
+   * @param closeable
+   */
+  public synchronized void setCloseable(C closeable) {
+    this.closeable = closeable;
+  }
+
+  /**
+   * Stop routine will close the closeable -if not null - and set the
+   * reference to null afterwards
+   * This operation does raise any exception on the close, though it does
+   * record it
+   */
+  @Override
+  protected void serviceStop() {
+    C target = getCloseable();
+    if (target != null) {
+      try {
+        target.close();
+      } catch (IOException ioe) {
+        noteFailure(ioe);
+      }
+      setCloseable(null);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
new file mode 100644
index 0000000..352be49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.slider.core.main.ServiceLaunchException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Service wrapper for an external program that is launched and can/will terminate.
+ * This service is notified when the subprocess terminates, and stops itself 
+ * and converts a non-zero exit code into a failure exception.
+ * 
+ * <p>
+ * Key Features:
+ * <ol>
+ *   <li>The property {@link #executionTimeout} can be set to set a limit
+ *   on the duration of a process</li>
+ *   <li>Output is streamed to the output logger provided</li>.
+ *   <li>The most recent lines of output are saved to a linked list</li>.
+ *   <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent}, is raised on the start
+ *   and finish of a process.</li>
+ * </ol>
+ *
+ * Usage:
+ * <p></p>
+ * The service can be built in the constructor, {@link #ForkedProcessService(String, Map, List)},
+ * or have its simple constructor used to instantiate the service, then the 
+ * {@link #build(Map, List)} command used to define the environment variables
+ * and list of commands to execute. One of these two options MUST be exercised
+ * before calling the services's {@link #start()} method.
+ * <p></p>
+ * The forked process is executed in the service's {@link #serviceStart()} method;
+ * if still running when the service is stopped, {@link #serviceStop()} will
+ * attempt to stop it.
+ * <p></p>
+ * 
+ * The service delegates process execution to {@link LongLivedProcess},
+ * receiving callbacks via the {@link LongLivedProcessLifecycleEvent}.
+ * When the service receives a callback notifying that the process has completed,
+ * it calls its {@link #stop()} method. If the error code was non-zero, 
+ * the service is logged as having failed.
+ */
+public class ForkedProcessService
+    extends WorkflowExecutorService<ExecutorService>
+    implements LongLivedProcessLifecycleEvent, Runnable {
+
+  /**
+   * Log for the forked master process
+   */
+  private static final Logger LOG =
+    LoggerFactory.getLogger(ForkedProcessService.class);
+
+  private final AtomicBoolean processTerminated = new AtomicBoolean(false);
+  private boolean processStarted = false;
+  private LongLivedProcess process;
+  private int executionTimeout = -1;
+  private int timeoutCode = 1;
+  /** 
+  log to log to; defaults to this service log
+   */
+  private Logger processLog = LOG;
+  
+  /**
+   * Exit code set when the spawned process exits
+   */
+  private AtomicInteger exitCode = new AtomicInteger(0);
+
+  /**
+   * Create an instance of the service
+   * @param name a name
+   */
+  public ForkedProcessService(String name) {
+    super(name);
+  }
+
+  /**
+   * Create an instance of the service,  set up the process
+   * @param name a name
+   * @param commandList list of commands is inserted on the front
+   * @param env environment variables above those generated by
+   * @throws IOException IO problems
+   */
+  public ForkedProcessService(String name,
+      Map<String, String> env,
+      List<String> commandList) throws IOException {
+    super(name);
+    build(env, commandList);
+  }
+
+  @Override //AbstractService
+  protected void serviceStart() throws Exception {
+    if (process == null) {
+      throw new ServiceStateException("Process not yet configured");
+    }
+    //now spawn the process -expect updates via callbacks
+    process.start();
+  }
+
+  @Override //AbstractService
+  protected void serviceStop() throws Exception {
+    completed();
+    stopForkedProcess();
+  }
+
+  private void stopForkedProcess() {
+    if (process != null) {
+      process.stop();
+    }
+  }
+
+  /**
+   * Set the process log. This may be null for "do not log"
+   * @param processLog process log
+   */
+  public void setProcessLog(Logger processLog) {
+    this.processLog = processLog;
+    process.setProcessLog(processLog);
+  }
+
+  /**
+   * Set the timeout by which time a process must have finished -or -1 for forever
+   * @param timeout timeout in milliseconds
+   */
+  public void setTimeout(int timeout, int code) {
+    this.executionTimeout = timeout;
+    this.timeoutCode = code;
+  }
+
+  /**
+   * Build the process to execute when the service is started
+   * @param commandList list of commands is inserted on the front
+   * @param env environment variables above those generated by
+   * @throws IOException IO problems
+   */
+  public void build(Map<String, String> env,
+                    List<String> commandList)
+      throws IOException {
+    assert process == null;
+
+    process = new LongLivedProcess(getName(), processLog, commandList);
+    process.setLifecycleCallback(this);
+    //set the env variable mapping
+    process.putEnvMap(env);
+  }
+
+  @Override // notification from executed process
+  public synchronized void onProcessStarted(LongLivedProcess process) {
+    LOG.debug("Process has started");
+    processStarted = true;
+    if (executionTimeout > 0) {
+      setExecutor(ServiceThreadFactory.singleThreadExecutor(getName(), true));
+      execute(this);
+    }
+  }
+
+  @Override  // notification from executed process
+  public void onProcessExited(LongLivedProcess process,
+      int uncorrected,
+      int code) {
+    try {
+      synchronized (this) {
+        completed();
+        //note whether or not the service had already stopped
+        LOG.debug("Process has exited with exit code {}", code);
+        if (code != 0) {
+          reportFailure(code, getName() + " failed with code " + code);
+        }
+      }
+    } finally {
+      stop();
+    }
+  }
+
+  private void reportFailure(int code, String text) {
+    //error
+    ServiceLaunchException execEx = new ServiceLaunchException(code, text);
+    LOG.debug("Noting failure", execEx);
+    noteFailure(execEx);
+  }
+
+  /**
+   * handle timeout response by escalating it to a failure
+   */
+  @Override
+  public void run() {
+    try {
+      synchronized (processTerminated) {
+        if (!processTerminated.get()) {
+          processTerminated.wait(executionTimeout);
+        }
+      }
+
+    } catch (InterruptedException e) {
+      //assume signalled; exit
+    }
+    //check the status; if the marker isn't true, bail
+    if (!processTerminated.getAndSet(true)) {
+      LOG.info("process timeout: reporting error code {}", timeoutCode);
+
+      //timeout
+      if (isInState(STATE.STARTED)) {
+        //trigger a failure
+        stopForkedProcess();
+      }
+      reportFailure(timeoutCode, getName() + ": timeout after " + executionTimeout
+                   + " millis: exit code =" + timeoutCode);
+    }
+  }
+
+  /**
+   * Note the process as having completed.
+   * The process marked as terminated
+   * -and anything synchronized on <code>processTerminated</code>
+   * is notified
+   */
+  protected void completed() {
+    processTerminated.set(true);
+    synchronized (processTerminated) {
+      processTerminated.notify();
+    }
+  }
+
+  public boolean isProcessTerminated() {
+    return processTerminated.get();
+  }
+
+  public synchronized boolean isProcessStarted() {
+    return processStarted;
+  }
+
+  /**
+   * Is a process running: between started and terminated
+   * @return true if the process is up.
+   */
+  public synchronized boolean isProcessRunning() {
+    return processStarted && !isProcessTerminated();
+  }
+
+
+  public Integer getExitCode() {
+    return process.getExitCode();
+  }
+  
+  public int getExitCodeSignCorrected() {
+    Integer exitCode = process.getExitCodeSignCorrected();
+    if (exitCode == null) return -1;
+    return exitCode;
+  }
+
+  /**
+   * Get the recent output from the process, or [] if not defined
+   * @return a possibly empty list
+   */
+  public List<String> getRecentOutput() {
+    return process != null
+           ? process.getRecentOutput()
+           : new LinkedList<String>();
+  }
+
+  /**
+   * Get the recent output from the process, or [] if not defined
+   *
+   * @param finalOutput flag to indicate "wait for the final output of the process"
+   * @param duration the duration, in ms, 
+   * to wait for recent output to become non-empty
+   * @return a possibly empty list
+   */
+  public List<String> getRecentOutput(boolean finalOutput, int duration) {
+    if (process == null) {
+      return new LinkedList<>();
+    }
+    return process.getRecentOutput(finalOutput, duration);
+  }
+  
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message