hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject [12/15] hadoop git commit: YARN-6903. Yarn-native-service framework core rewrite. Contributed by Jian He
Date Mon, 14 Aug 2017 21:13:50 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
new file mode 100644
index 0000000..a4a0a15
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -0,0 +1,487 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.ContainerFailureTracker;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.servicemonitor.MonitorUtils;
+import org.apache.slider.server.servicemonitor.Probe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STARTED;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.STOP;
+import static org.apache.hadoop.yarn.service.component.ComponentState.*;
+import static org.apache.slider.api.ResourceKeys.CONTAINER_FAILURE_THRESHOLD;
+
+public class Component implements EventHandler<ComponentEvent> {
+  private static final Logger LOG = LoggerFactory.getLogger(Component.class);
+
+  private org.apache.slider.api.resource.Component componentSpec;
+  private long allocateId;
+  private Priority priority;
+  private ServiceMetrics componentMetrics;
+  private ServiceScheduler scheduler;
+  private ServiceContext context;
+  private AMRMClientAsync<ContainerRequest> amrmClient;
+  private AtomicLong instanceIdCounter = new AtomicLong();
+  private Map<ComponentInstanceId, ComponentInstance> compInstances =
+      new ConcurrentHashMap<>();
+  // component instances to be assigned with a container
+  private List<ComponentInstance> pendingInstances = new LinkedList<>();
+  private ContainerFailureTracker failureTracker;
+  private Probe probe;
+  private final ReentrantReadWriteLock.ReadLock readLock;
+  private final ReentrantReadWriteLock.WriteLock writeLock;
+  public int maxContainerFailurePerComp;
+  // The number of containers failed since last reset. This excludes preempted,
+  // disk_failed containers etc. This will be reset to 0 periodically.
+  public volatile int currentContainerFailure;
+
+  private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
+      stateMachine;
+  private AsyncDispatcher compInstanceDispatcher;
+  private static final StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>
+      stateMachineFactory =
+      new StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>(
+          INIT)
+           // INIT will only got to FLEXING
+          .addTransition(INIT, EnumSet.of(STABLE, FLEXING),
+              FLEX, new FlexComponentTransition())
+
+          // container allocated by RM
+          .addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
+              new ContainerAllocatedTransition())
+          // container launched on NM
+          .addTransition(FLEXING, EnumSet.of(STABLE, FLEXING),
+              CONTAINER_STARTED, new ContainerStartedTransition())
+          // container failed while flexing
+          .addTransition(FLEXING, FLEXING, CONTAINER_COMPLETED,
+              new ContainerCompletedTransition())
+          // Flex while previous flex is still in progress
+          .addTransition(FLEXING, EnumSet.of(FLEXING), FLEX,
+              new FlexComponentTransition())
+
+          // container failed while stable
+          .addTransition(STABLE, FLEXING, CONTAINER_COMPLETED,
+              new ContainerCompletedTransition())
+          // Ignore surplus container
+          .addTransition(STABLE, STABLE, CONTAINER_ALLOCATED,
+              new ContainerAllocatedTransition())
+          // Flex by user
+          // For flex up, go to FLEXING state
+          // For flex down, go to STABLE state
+          .addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
+              FLEX, new FlexComponentTransition())
+          .installTopology();
+
+  public Component(org.apache.slider.api.resource.Component component,
+      long allocateId, ServiceContext context) {
+    this.allocateId = allocateId;
+    this.priority = Priority.newInstance((int) allocateId);
+    this.componentSpec = component;
+    componentMetrics = ServiceMetrics.register(component.getName(),
+        "Metrics for component " + component.getName());
+    componentMetrics
+        .tag("type", "Metrics type [component or service]", "component");
+    this.scheduler = context.scheduler;
+    this.context = context;
+    amrmClient = scheduler.getAmRMClient();
+    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    this.readLock = lock.readLock();
+    this.writeLock = lock.writeLock();
+    this.stateMachine = stateMachineFactory.make(this);
+    compInstanceDispatcher = scheduler.getCompInstanceDispatcher();
+    failureTracker =
+        new ContainerFailureTracker(context, this);
+    probe = MonitorUtils.getProbe(componentSpec.getReadinessCheck());
+    maxContainerFailurePerComp = componentSpec.getConfiguration()
+        .getPropertyInt(CONTAINER_FAILURE_THRESHOLD, 10);
+    createNumCompInstances(component.getNumberOfContainers());
+  }
+
+  private void createNumCompInstances(long count) {
+    for (int i = 0; i < count; i++) {
+      createOneCompInstance();
+    }
+  }
+
+  private void createOneCompInstance() {
+    ComponentInstanceId id =
+        new ComponentInstanceId(instanceIdCounter.getAndIncrement(),
+            componentSpec.getName());
+    ComponentInstance instance = new ComponentInstance(this, id);
+    compInstances.put(id, instance);
+    pendingInstances.add(instance);
+  }
+
+  private static class FlexComponentTransition implements
+      MultipleArcTransition<Component, ComponentEvent, ComponentState> {
+    // For flex up, go to FLEXING state
+    // For flex down, go to STABLE state
+    @Override
+    public ComponentState transition(Component component,
+        ComponentEvent event) {
+      component.setDesiredContainers((int)event.getDesired());
+      if (!component.areDependenciesReady()) {
+        LOG.info("[FLEX COMPONENT {}]: Flex deferred because dependencies not"
+            + " satisfied.", component.getName());
+        return component.getState();
+      }
+      if (component.getState() == INIT) {
+        // This happens on init
+        LOG.info("[INIT COMPONENT " + component.getName() + "]: " + event
+            .getDesired() + " instances.");
+        component.requestContainers(event.getDesired());
+        return FLEXING;
+      }
+      long before = component.getComponentSpec().getNumberOfContainers();
+      long delta = event.getDesired() - before;
+      component.getComponentSpec().setNumberOfContainers(event.getDesired());
+      if (delta > 0) {
+        // Scale up
+        LOG.info("[FLEX UP COMPONENT " + component.getName() + "]: scaling up from "
+                + before + " to " + event.getDesired());
+        component.requestContainers(delta);
+        component.createNumCompInstances(delta);
+        return FLEXING;
+      } else if (delta < 0){
+        delta = 0 - delta;
+        // scale down
+        LOG.info("[FLEX DOWN COMPONENT " + component.getName()
+            + "]: scaling down from " + before + " to " + event.getDesired());
+        List<ComponentInstance> list =
+            new ArrayList<>(component.compInstances.values());
+
+        // sort in Most recent -> oldest order, destroy most recent ones.
+        Collections.sort(list, Collections.reverseOrder());
+        for (int i = 0; i < delta; i++) {
+          ComponentInstance instance = list.get(i);
+          // remove the instance
+          component.compInstances.remove(instance.getCompInstanceId());
+          component.pendingInstances.remove(instance);
+          component.componentMetrics.containersFailed.incr();
+          component.componentMetrics.containersRunning.decr();
+          // decrement id counter
+          component.instanceIdCounter.decrementAndGet();
+          instance.destroy();
+        }
+        return STABLE;
+      } else {
+        LOG.info("[FLEX COMPONENT " + component.getName() + "]: already has " +
+            event.getDesired() + " instances, ignoring");
+        return STABLE;
+      }
+    }
+  }
+
+  private static class ContainerAllocatedTransition extends BaseTransition {
+    @Override
+    public void transition(Component component, ComponentEvent event) {
+      component.assignContainerToCompInstance(event.getContainer());
+    }
+  }
+
+  private static class ContainerStartedTransition implements
+      MultipleArcTransition<Component,ComponentEvent,ComponentState> {
+
+    @Override public ComponentState transition(Component component,
+        ComponentEvent event) {
+      component.compInstanceDispatcher.getEventHandler().handle(
+          new ComponentInstanceEvent(event.getInstance().getContainerId(),
+              STARTED));
+      component.incRunningContainers();
+      return checkIfStable(component);
+    }
+  }
+
+  private static ComponentState checkIfStable(Component component) {
+    // if desired == running
+    if (component.componentMetrics.containersRunning.value() == component
+        .getComponentSpec().getNumberOfContainers()) {
+      return STABLE;
+    } else {
+      return FLEXING;
+    }
+  }
+
+  private static class ContainerCompletedTransition extends BaseTransition {
+    @Override
+    public void transition(Component component, ComponentEvent event) {
+      component.updateMetrics(event.getStatus());
+
+      // add back to pending list
+      component.pendingInstances.add(event.getInstance());
+      LOG.info(
+          "[COMPONENT {}]: {} completed, num pending comp instances increased to {}.",
+          component.getName(), event.getStatus().getContainerId(),
+          component.pendingInstances.size());
+      component.compInstanceDispatcher.getEventHandler().handle(
+          new ComponentInstanceEvent(event.getStatus().getContainerId(),
+              STOP).setStatus(event.getStatus()));
+    }
+  }
+
+  public ServiceMetrics getCompMetrics () {
+    return componentMetrics;
+  }
+
+  private void assignContainerToCompInstance(Container container) {
+    if (pendingInstances.size() == 0) {
+      LOG.info(
+          "[COMPONENT {}]: No pending component instance left, release surplus container {}",
+          getName(), container.getId());
+      scheduler.getAmRMClient().releaseAssignedContainer(container.getId());
+      componentMetrics.surplusContainers.incr();
+      scheduler.getServiceMetrics().surplusContainers.incr();
+      return;
+    }
+    ComponentInstance instance = pendingInstances.remove(0);
+    LOG.info(
+        "[COMPONENT {}]: {} allocated, num pending component instances reduced to {}",
+        getName(), container.getId(), pendingInstances.size());
+    instance.setContainer(container);
+    scheduler.addLiveCompInstance(container.getId(), instance);
+    LOG.info(
+        "[COMPONENT {}]: Assigned {} to component instance {} and launch on host {} ",
+        getName(), container.getId(), instance.getCompInstanceName(),
+        container.getNodeId());
+    scheduler.getContainerLaunchService()
+        .launchCompInstance(scheduler.getApp(), instance, container);
+  }
+
+  @SuppressWarnings({ "unchecked" })
+  public void requestContainers(long count) {
+    Resource resource = Resource
+        .newInstance(componentSpec.getResource().getMemoryMB(),
+            componentSpec.getResource().getCpus());
+
+    for (int i = 0; i < count; i++) {
+      //TODO Once YARN-5468 is done, use that for anti-affinity
+      ContainerRequest request =
+          ContainerRequest.newBuilder().capability(resource).priority(priority)
+              .allocationRequestId(allocateId).relaxLocality(true).build();
+      amrmClient.addContainerRequest(request);
+    }
+  }
+
+  private void setDesiredContainers(int n) {
+    int delta = n - scheduler.getServiceMetrics().containersDesired.value();
+    if (delta > 0) {
+      scheduler.getServiceMetrics().containersDesired.incr(delta);
+    } else {
+      scheduler.getServiceMetrics().containersDesired.decr(delta);
+    }
+    componentMetrics.containersDesired.set(n);
+  }
+
+
+
+  private void updateMetrics(ContainerStatus status) {
+    switch (status.getExitStatus()) {
+    case SUCCESS:
+      componentMetrics.containersSucceeded.incr();
+      scheduler.getServiceMetrics().containersSucceeded.incr();
+      return;
+    case PREEMPTED:
+      componentMetrics.containersPreempted.incr();
+      scheduler.getServiceMetrics().containersPreempted.incr();
+      break;
+    case DISKS_FAILED:
+      componentMetrics.containersDiskFailure.incr();
+      scheduler.getServiceMetrics().containersDiskFailure.incr();
+      break;
+    default:
+      break;
+    }
+
+    // containersFailed include preempted, disks_failed etc.
+    componentMetrics.containersFailed.incr();
+    scheduler.getServiceMetrics().containersFailed.incr();
+
+    // dec running container
+    decRunningContainers();
+
+    if (Apps.shouldCountTowardsNodeBlacklisting(status.getExitStatus())) {
+      String host = scheduler.getLiveInstances().get(status.getContainerId())
+          .getNodeId().getHost();
+      failureTracker.incNodeFailure(host);
+      currentContainerFailure++ ;
+    }
+  }
+
+  public boolean areDependenciesReady() {
+    List<String> dependencies = componentSpec.getDependencies();
+    if (SliderUtils.isEmpty(dependencies)) {
+      return true;
+    }
+    for (String dependency : dependencies) {
+      Component dependentComponent =
+          scheduler.getAllComponents().get(dependency);
+      if (dependentComponent == null) {
+        LOG.error("Couldn't find dependency {} for {} (should never happen)",
+            dependency, getName());
+        continue;
+      }
+      if (dependentComponent.getNumReadyInstances() < dependentComponent
+          .getNumDesiredInstances()) {
+        LOG.info("[COMPONENT {}]: Dependency {} not satisfied, only {} of {}"
+                + " instances are ready.", getName(), dependency,
+            dependentComponent.getNumReadyInstances(),
+            dependentComponent.getNumDesiredInstances());
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private void incRunningContainers() {
+    componentMetrics.containersRunning.incr();
+    scheduler.getServiceMetrics().containersRunning.incr();
+  }
+
+  public void incContainersReady() {
+    componentMetrics.containersReady.incr();
+  }
+
+  public void decContainersReady() {
+    componentMetrics.containersReady.decr();
+  }
+
+  private void decRunningContainers() {
+    componentMetrics.containersRunning.decr();
+    scheduler.getServiceMetrics().containersRunning.decr();
+  }
+
+  public int getNumReadyInstances() {
+    return componentMetrics.containersReady.value();
+  }
+
+  public int getNumRunningInstances() {
+    return componentMetrics.containersRunning.value();
+  }
+
+  public int getNumDesiredInstances() {
+    return componentMetrics.containersDesired.value();
+  }
+
+  public Map<ComponentInstanceId, ComponentInstance> getAllComponentInstances() {
+    return compInstances;
+  }
+
+  public org.apache.slider.api.resource.Component getComponentSpec() {
+    return this.componentSpec;
+  }
+
+  public void resetCompFailureCount() {
+    LOG.info("[COMPONENT {}]: Reset container failure count from {} to 0.",
+        getName(), currentContainerFailure);
+    currentContainerFailure = 0;
+    failureTracker.resetContainerFailures();
+  }
+
+  public Probe getProbe() {
+    return probe;
+  }
+
+  public Priority getPriority() {
+    return priority;
+  }
+
+  public String getName () {
+    return componentSpec.getName();
+  }
+
+  public ComponentState getState() {
+    this.readLock.lock();
+
+    try {
+      return this.stateMachine.getCurrentState();
+    } finally {
+      this.readLock.unlock();
+    }
+  }
+  public ServiceScheduler getScheduler() {
+    return scheduler;
+  }
+
+  @Override
+  public void handle(ComponentEvent event) {
+    try {
+      writeLock.lock();
+      ComponentState oldState = getState();
+      try {
+        stateMachine.doTransition(event.getType(), event);
+      } catch (InvalidStateTransitionException e) {
+        LOG.error("Invalid event " + event.getType() +
+            " at " + oldState + " for component " + componentSpec.getName(), e);
+      }
+      if (oldState != getState()) {
+        LOG.info("[COMPONENT {}] Transitioned from {} to {} on {} event.",
+            componentSpec.getName(), oldState, getState(), event.getType());
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private static class BaseTransition implements
+      SingleArcTransition<Component, ComponentEvent> {
+
+    @Override public void transition(Component component,
+        ComponentEvent event) {
+    }
+  }
+
+  public ServiceContext getContext() {
+    return context;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
new file mode 100644
index 0000000..ed892dd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+
+public class ComponentEvent extends AbstractEvent<ComponentEventType> {
+  private long desired;
+  private final String name;
+  private final ComponentEventType type;
+  private Container container;
+  private ComponentInstance instance;
+  private ContainerStatus status;
+
+  public ComponentEvent(String name, ComponentEventType type) {
+    super(type);
+    this.name = name;
+    this.type = type;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public ComponentEventType getType() {
+    return type;
+  }
+
+  public long getDesired() {
+    return desired;
+  }
+
+  public ComponentEvent setDesired(long desired) {
+    this.desired = desired;
+    return this;
+  }
+
+  public Container getContainer() {
+    return container;
+  }
+
+  public ComponentEvent setContainer(Container container) {
+    this.container = container;
+    return this;
+  }
+
+  public ComponentInstance getInstance() {
+    return instance;
+  }
+
+  public ComponentEvent setInstance(ComponentInstance instance) {
+    this.instance = instance;
+    return this;
+  }
+
+  public ContainerStatus getStatus() {
+    return status;
+  }
+
+  public ComponentEvent setStatus(ContainerStatus status) {
+    this.status = status;
+    return this;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
new file mode 100644
index 0000000..6729699
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentEventType {
+  FLEX,
+  CONTAINER_ALLOCATED,
+  CONTAINER_STARTED,
+  CONTAINER_COMPLETED
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
new file mode 100644
index 0000000..a5f9ff4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+public enum ComponentState {
+  INIT,
+  FLEXING,
+  STABLE
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
new file mode 100644
index 0000000..d63c1a4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderExitCodes.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+import org.apache.slider.core.main.LauncherExitCodes;
+
+public interface SliderExitCodes extends LauncherExitCodes {
+
+  /**
+   * starting point for exit codes; not an exception itself
+   */
+  int _EXIT_CODE_BASE =           64;
+
+  /**
+   * service entered the failed state: {@value}
+   */
+  int EXIT_YARN_SERVICE_FAILED =  65;
+
+  /**
+   * service was killed: {@value}
+   */
+  int EXIT_YARN_SERVICE_KILLED =  66;
+
+  /**
+   * timeout on monitoring client: {@value}
+   */
+  int EXIT_TIMED_OUT =            67;
+
+  /**
+   * service finished with an error: {@value}
+   */
+  int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
+
+  /**
+   * the application instance is unknown: {@value}
+   */
+  int EXIT_UNKNOWN_INSTANCE =     69;
+
+  /**
+   * the application instance is in the wrong state for that operation: {@value}
+   */
+  int EXIT_BAD_STATE =            70;
+
+  /**
+   * A spawned master process failed 
+   */
+  int EXIT_PROCESS_FAILED =       71;
+
+  /**
+   * The instance failed -too many containers were
+   * failing or some other threshold was reached
+   */
+  int EXIT_DEPLOYMENT_FAILED =    72;
+
+  /**
+   * The application is live -and the requested operation
+   * does not work if the cluster is running
+   */
+  int EXIT_APPLICATION_IN_USE =   73;
+
+  /**
+   * There already is an application instance of that name
+   * when an attempt is made to create a new instance
+   */
+  int EXIT_INSTANCE_EXISTS =      75;
+
+  /**
+   * Exit code when the configurations in valid/incomplete: {@value}
+   */
+  int EXIT_BAD_CONFIGURATION =    77;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
new file mode 100644
index 0000000..e1687d2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderKeys.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+/**
+ * Keys and various constants for Slider
+ */
+public interface SliderKeys extends SliderXmlConfKeys {
+
+  /**
+   * This is the name of the slider appmaster in configurations :{@value}
+   */
+  String COMPONENT_AM = "slider-appmaster";
+  
+  /**
+   * Slider role is "special":{@value}
+   */
+  int ROLE_AM_PRIORITY_INDEX = 0;
+  
+  
+  /**
+   * The path under which cluster and temp data are stored
+   * {@value}
+   */
+  String SLIDER_BASE_DIRECTORY = ".slider";
+
+  /**
+   * The paths under which Slider AM dependency libraries are stored
+   */
+  String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep";
+  String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp";
+  String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider";
+  String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider-dep";
+  String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz";
+  String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755";
+
+  /**
+   * 
+   */
+  String HDP_VERSION_PROP_NAME = "HDP_VERSION";
+
+  /**
+   *  name of the relative path to expaned an image into:  {@value}.
+   *  The title of this path is to help people understand it when
+   *  they see it in their error messages
+   */
+  String LOCAL_TARBALL_INSTALL_SUBDIR = "expandedarchive";
+
+
+  /**
+   * Application type for YARN  {@value}
+   */
+  String APP_TYPE = "org-apache-slider";
+
+  /**
+   * A component type for an external app that has been predefined using the
+   * slider build command
+   */
+  String COMPONENT_SEPARATOR = "-";
+
+  /**
+   * A component type for a client component
+   */
+  String COMPONENT_TYPE_CLIENT = "client";
+
+  /**
+   * Key for application version.
+   */
+  String APP_VERSION_UNKNOWN = "awaiting heartbeat...";
+
+  /**
+   * Keys for application container specific properties, like release timeout
+   */
+  String APP_CONTAINER_RELEASE_TIMEOUT = "site.global.app_container.release_timeout_secs";
+
+  /**
+   * Subdirectories of HDFS cluster dir.
+   */
+  String DATA_DIR_NAME = "data";
+  String HISTORY_DIR_NAME = "history";
+  String HISTORY_FILENAME_SUFFIX = "json";
+  String HISTORY_FILENAME_PREFIX = "rolehistory-";
+  String KEYTAB_DIR = "keytabs";
+  String RESOURCE_DIR = "resources";
+
+  /**
+   * Filename pattern is required to save in strict temporal order.
+   * Important: older files must sort less-than newer files when using
+   * case-sensitive name sort.
+   */
+  String HISTORY_FILENAME_CREATION_PATTERN = HISTORY_FILENAME_PREFIX +"%016x."+
+                                    HISTORY_FILENAME_SUFFIX;
+  /**
+   * The posix regexp used to locate this 
+   */
+  String HISTORY_FILENAME_MATCH_PATTERN = HISTORY_FILENAME_PREFIX +"[0-9a-f]+\\."+
+                                    HISTORY_FILENAME_SUFFIX;
+    /**
+   * The posix regexp used to locate this 
+   */
+  String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
+                                    HISTORY_FILENAME_SUFFIX;
+
+  String CLUSTER_DIRECTORY = "cluster";
+
+  /**
+   * JVM property to define the slider lib directory;
+   * this is set by the slider script: {@value}
+   */
+  String PROPERTY_LIB_DIR = "slider.libdir";
+
+  /**
+   * name of generated dir for this conf: {@value}
+   */
+  String SUBMITTED_CONF_DIR = "conf";
+
+  /**
+   * Slider AM log4j file name : {@value}
+   */
+  String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
+
+  /**
+   * Log4j sysprop to name the resource :{@value}
+   */
+  String SYSPROP_LOG4J_CONFIGURATION = "log4j.configuration";
+
+  /**
+   * sysprop for Slider AM log4j directory :{@value}
+   */
+  String SYSPROP_LOG_DIR = "LOG_DIR";
+
+  /**
+   * name of the Slider client resource
+   * loaded when the service is loaded.
+   */
+  String SLIDER_CLIENT_XML = "slider-client.xml";
+
+  /**
+   * The name of the resource to put on the classpath
+   */
+  String SLIDER_SERVER_XML = "slider-server.xml";
+
+  String TMP_DIR_PREFIX = "tmp";
+
+  /**
+   * Store the default app definition, e.g. metainfo file or content of a folder
+   */
+  String APP_DEF_DIR = "appdef";
+  /**
+   * Store additional app defs - co-processors
+   */
+  String ADDONS_DIR = "addons";
+
+  String SLIDER_JAR = "slider-core.jar";
+
+  String STDOUT_AM = "slider-out.txt";
+  String STDERR_AM = "slider-err.txt";
+
+  String HADOOP_USER_NAME = "HADOOP_USER_NAME";
+
+  /**
+   * Name of the AM filter to use: {@value}
+   */
+  String AM_FILTER_NAME =
+      "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer";
+
+  String YARN_CONTAINER_PATH = "/node/container/";
+
+  String APP_CONF_DIR = "conf";
+
+  String APP_LIB_DIR = "lib";
+
+  String OUT_FILE = "stdout.txt";
+  String ERR_FILE = "stderr.txt";
+
+  String QUICK_LINKS = "quicklinks";
+
+  String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
new file mode 100644
index 0000000..523e08d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/SliderXmlConfKeys.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+/**
+ * These are the keys that can be added to <code>conf/slider-client.xml</code>.
+ */
+public interface SliderXmlConfKeys {
+  String PREFIX_PROVIDER = "slider.provider";
+  /**
+   * pattern to identify a provider
+   * {@value}
+   */
+  String KEY_PROVIDER = PREFIX_PROVIDER + ".%s";
+
+  /**
+   * conf option set to point to where the config came from
+   * {@value}
+   */
+  String KEY_TEMPLATE_ORIGIN = "slider.template.origin";
+
+  /**
+   * Original name for the default FS. This is still 
+   * expected by applications deployed
+   */
+  String FS_DEFAULT_NAME_CLASSIC = "fs.default.name";
+
+  /**
+   * Slider principal
+   */
+  String KEY_KERBEROS_PRINCIPAL = "slider.kerberos.principal";
+
+  /**
+   * Name of the property for ACLs for Slider AM.
+   * {@value}
+   */
+  String KEY_PROTOCOL_ACL = "slider.security.protocol.acl";
+
+  /**
+   * Limit on restarts for the AM
+   * {@value}
+   */
+  String KEY_AM_RESTART_LIMIT = "slider.yarn.restart.limit";
+
+  /**
+   * queue name, by default let YARN pick the queue
+   */
+  String KEY_YARN_QUEUE = "slider.yarn.queue";
+  String DEFAULT_YARN_QUEUE = null;
+
+  /**
+   * default priority
+   */
+  String KEY_YARN_QUEUE_PRIORITY = "slider.yarn.queue.priority";
+  int DEFAULT_YARN_QUEUE_PRIORITY = 1;
+
+
+  String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory";
+  long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
+
+  /**
+   * The slider base path: {@value}
+   * Defaults to HomeDir/.slider
+   */
+  String KEY_SLIDER_BASE_PATH = "slider.base.path";
+
+
+  /**
+   * Option for the permissions for the cluster directory itself: {@value}
+   */
+  String CLUSTER_DIRECTORY_PERMISSIONS =
+    "slider.cluster.directory.permissions";
+
+  /**
+   * Default value for the permissions :{@value}
+   */
+  String DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS = "750";
+
+  /**
+   * 
+   * Option for the permissions for the data directory itself: {@value}
+   */
+  String DATA_DIRECTORY_PERMISSIONS = "slider.data.directory.permissions";
+
+  /**
+   * Default value for the data directory permissions: {@value}
+   */
+  String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
+
+  String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
+      "ipc.client.fallback-to-simple-auth-allowed";
+  String HADOOP_HTTP_FILTER_INITIALIZERS =
+      "hadoop.http.filter.initializers";
+  String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
+  String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+  /** Declare that a keytab must be provided */
+  String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
+  String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
+  String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+  String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
+  String KEY_SECURITY_ENABLED = "site.global.security_enabled";
+
+  /**
+   * Set to disable server-side checks for python, openssl &c.
+   * This should only be set for testing
+   */
+  String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED =
+      "slider.am.dependency.checks.disabled";
+
+  /**
+   * The path to the python executable utilized to launch the agent.
+   */
+  String PYTHON_EXECUTABLE_PATH = "agent.python.exec.path";
+
+  /**
+   * Flag to enable the insecure AM filter: {@value}
+   */
+  String X_DEV_INSECURE_WS = "slider.feature.ws.insecure";
+
+  /**
+   * Flag to indicate the insecure AM filter is enabled by default: {@value}.
+   */
+  boolean X_DEV_INSECURE_DEFAULT = false;
+
+
+  /**
+   * Flag to indicate the insecure AM filter is required for
+   * complex REST Verbs: {@value}.
+   * When Slider switches to being Hadoop 2.7+ only, this flag
+   * can be set to false
+   */
+  boolean X_DEV_INSECURE_REQUIRED = true;
+
+  /**
+   *
+   */
+  String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED =
+      "slider.ipc.client.retry.enabled";
+  boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
+  String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
+      "slider.ipc.client.retry.policy.spec";
+  String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
+      "10000,6,60000,10"; //t1,n1,t2,n2,... 
+
+  String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
+
+  /**
+   * From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY}
+   */
+  String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal";
+
+  String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
+
+  //Delegation token related keys
+  String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY
+      = "dfs.namenode.delegation.key.update-interval";
+  long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 *
+      1000; // 1 day
+  String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY
+      = "dfs.namenode.delegation.token.renew-interval";
+  long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 *
+      1000;  // 1 day
+  String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY
+      = "dfs.namenode.delegation.token.max-lifetime";
+  long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 *
+      1000; // 7 days
+  String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY
+      = "dfs.namenode.delegation.token.always-use"; // for tests
+  boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
+  String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
+  String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
+
+  String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary";
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java
new file mode 100644
index 0000000..4fda686
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConfKeys.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.conf;
+
+public interface YarnServiceConfKeys {
+
+  // Retry settings for the ServiceClient to talk to Service AppMaster
+  String CLIENT_AM_RETRY_MAX_WAIT_MS = "yarn.service.client-am.retry.max-wait-ms";
+  String CLIENT_AM_RETRY_MAX_INTERVAL_MS = "yarn.service.client-am.retry-interval-ms";
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
new file mode 100644
index 0000000..33e33a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.impl.pb.service.ClientAMProtocolPB;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+
+public class ClientAMProtocolPBClientImpl
+    implements ClientAMProtocol, Closeable {
+
+  private ClientAMProtocolPB proxy;
+
+  public ClientAMProtocolPBClientImpl(long clientVersion,
+      InetSocketAddress addr, Configuration conf) throws IOException {
+    RPC.setProtocolEngine(conf, ClientAMProtocolPB.class,
+        ProtobufRpcEngine.class);
+    proxy = RPC.getProxy(ClientAMProtocolPB.class, clientVersion, addr, conf);
+
+  }
+
+  @Override public FlexComponentsResponseProto flexComponents(
+      FlexComponentsRequestProto request) throws IOException, YarnException {
+    try {
+      return proxy.flexComponents(null, request);
+    } catch (ServiceException e) {
+      RPCUtil.unwrapAndThrowException(e);
+    }
+    return null;
+  }
+
+  @Override
+  public GetStatusResponseProto getStatus(GetStatusRequestProto request)
+      throws IOException, YarnException {
+    try {
+      return proxy.getStatus(null, request);
+    } catch (ServiceException e) {
+      RPCUtil.unwrapAndThrowException(e);
+    }
+    return null;
+  }
+
+  @Override
+  public StopResponseProto stop(StopRequestProto requestProto)
+      throws IOException, YarnException {
+    try {
+      return proxy.stop(null, requestProto);
+    } catch (ServiceException e) {
+      RPCUtil.unwrapAndThrowException(e);
+    }
+    return null;
+  }
+
+  @Override public void close() {
+    if (this.proxy != null) {
+      RPC.stopProxy(this.proxy);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
new file mode 100644
index 0000000..6a9cd37
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPB.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol;
+
+@ProtocolInfo(
+    protocolName = "org.apache.hadoop.yarn.service.ClientAMProtocol",
+    protocolVersion = 1)
+public interface ClientAMProtocolPB extends
+    ClientAMProtocol.ClientAMProtocolService.BlockingInterface {
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
new file mode 100644
index 0000000..7100781
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.impl.pb.service;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+
+import java.io.IOException;
+
+public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
+
+  private ClientAMProtocol real;
+
+  public ClientAMProtocolPBServiceImpl(ClientAMProtocol impl) {
+    this.real = impl;
+  }
+
+  @Override
+  public FlexComponentsResponseProto flexComponents(RpcController controller,
+      FlexComponentsRequestProto request) throws ServiceException {
+    try {
+      return real.flexComponents(request);
+    } catch (IOException | YarnException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override public GetStatusResponseProto getStatus(RpcController controller,
+      GetStatusRequestProto request) throws ServiceException {
+    try {
+      return real.getStatus(request);
+    } catch (IOException | YarnException e) {
+      throw new ServiceException(e);
+    }
+  }
+
+  @Override
+  public org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto stop(
+      RpcController controller,
+      org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto request)
+      throws ServiceException {
+    try {
+      return real.stop(request);
+    } catch (IOException | YarnException e) {
+      throw new ServiceException(e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
new file mode 100644
index 0000000..bfe3bc9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/metrics/ServiceMetrics.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.metrics;
+
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+@Metrics(context = "yarn-native-service")
+public class ServiceMetrics implements MetricsSource {
+
+  @Metric("containers requested")
+  public MutableGaugeInt containersRequested;
+
+  @Metric("anti-affinity containers pending")
+  public MutableGaugeInt pendingAAContainers;
+
+  @Metric("containers running")
+  public MutableGaugeInt containersRunning;
+
+  @Metric("containers ready")
+  public MutableGaugeInt containersReady;
+
+  @Metric("containers desired")
+  public MutableGaugeInt containersDesired;
+
+  @Metric("containers succeeded")
+  public MutableGaugeInt containersSucceeded;
+
+  @Metric("containers failed")
+  public MutableGaugeInt containersFailed;
+
+  @Metric("containers preempted")
+  public MutableGaugeInt containersPreempted;
+
+  @Metric("containers exceeded limits")
+  public MutableGaugeInt containersLimitsExceeded;
+
+  @Metric("containers surplus")
+  public MutableGaugeInt surplusContainers;
+
+  @Metric("containers failed due to disk failure")
+  public MutableGaugeInt containersDiskFailure;
+
+  protected final MetricsRegistry registry;
+
+  public ServiceMetrics(MetricsInfo metricsInfo) {
+    registry = new MetricsRegistry(metricsInfo);
+  }
+
+  @Override
+  public void getMetrics(MetricsCollector collector, boolean all) {
+    registry.snapshot(collector.addRecord(registry.info()), all);
+  }
+
+  public static ServiceMetrics register(String name, String description) {
+    ServiceMetrics metrics = new ServiceMetrics(info(name, description));
+    DefaultMetricsSystem.instance().register(name, description, metrics);
+    return metrics;
+  }
+
+  public void tag(String name, String description, String value) {
+    registry.tag(name, description, value);
+  }
+
+  @Override public String toString() {
+    return "ServiceMetrics{"
+        + "containersRequested=" + containersRequested.value()
+        + ", pendingAAContainers=" + pendingAAContainers.value()
+        + ", containersRunning=" + containersRunning.value()
+        + ", containersDesired=" + containersDesired.value()
+        + ", containersSucceeded=" + containersSucceeded.value()
+        + ", containersFailed=" + containersFailed.value()
+        + ", containersPreempted=" + containersPreempted.value()
+        + ", surplusContainers=" + surplusContainers.value() + '}';
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
new file mode 100644
index 0000000..6c91a13
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class AbstractClientProvider {
+
+  public AbstractClientProvider() {
+  }
+
+  /**
+   * Generates a fixed format of application tags given one or more of
+   * application name, version and description. This allows subsequent query for
+   * an application with a name only, version only or description only or any
+   * combination of those as filters.
+   *
+   * @param appName name of the application
+   * @param appVersion version of the application
+   * @param appDescription brief description of the application
+   * @return
+   */
+  public static final Set<String> createApplicationTags(String appName,
+      String appVersion, String appDescription) {
+    Set<String> tags = new HashSet<>();
+    tags.add(SliderUtils.createNameTag(appName));
+    if (appVersion != null) {
+      tags.add(SliderUtils.createVersionTag(appVersion));
+    }
+    if (appDescription != null) {
+      tags.add(SliderUtils.createDescriptionTag(appDescription));
+    }
+    return tags;
+  }
+
+  /**
+   * Validate the artifact.
+   * @param artifact
+   */
+  public abstract void validateArtifact(Artifact artifact, FileSystem
+      fileSystem) throws IOException;
+
+  protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+      fileSystem) throws IOException;
+
+  /**
+   * Validate the config files.
+   * @param configFiles config file list
+   * @param fs file system
+   */
+  public void validateConfigFiles(List<ConfigFile> configFiles,
+      FileSystem fs) throws IOException {
+    Set<String> destFileSet = new HashSet<>();
+
+    for (ConfigFile file : configFiles) {
+      if (file.getType() == null) {
+        throw new IllegalArgumentException("File type is empty");
+      }
+
+      if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+          .isEmpty(file.getSrcFile())) {
+        throw new IllegalArgumentException(
+            "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+      }
+      if (!StringUtils.isEmpty(file.getSrcFile())) {
+        Path p = new Path(file.getSrcFile());
+        if (!fs.exists(p)) {
+          throw new IllegalArgumentException(
+              "Src_file does not exist for config file: " + file
+                  .getSrcFile());
+        }
+      }
+
+      if (StringUtils.isEmpty(file.getDestFile())) {
+        throw new IllegalArgumentException("Dest_file is empty.");
+      }
+
+      if (destFileSet.contains(file.getDestFile())) {
+        throw new IllegalArgumentException(
+            "Duplicated ConfigFile exists: " + file.getDestFile());
+      }
+      destFileSet.add(file.getDestFile());
+
+      java.nio.file.Path destPath = Paths.get(file.getDestFile());
+      if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
+        throw new IllegalArgumentException("Non-absolute dest_file has more " +
+            "than one path element");
+      }
+
+      // provider-specific validation
+      validateConfigFile(file, fs);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
new file mode 100644
index 0000000..472ee21
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService implements ProviderService,
+  SliderKeys {
+
+  protected static final Logger log =
+      LoggerFactory.getLogger(AbstractProviderService.class);
+
+  public abstract void processArtifact(AbstractLauncher launcher,
+      ComponentInstance compInstance, SliderFileSystem fileSystem,
+      Application application)
+      throws IOException;
+
+  public void buildContainerLaunchContext(AbstractLauncher launcher,
+      Application application, ComponentInstance instance,
+      SliderFileSystem fileSystem) throws IOException, SliderException {
+    Component component = instance.getComponent().getComponentSpec();;
+    processArtifact(launcher, instance, fileSystem, application);
+
+    ServiceContext context =
+        instance.getComponent().getScheduler().getContext();
+    // Generate tokens (key-value pair) for config substitution.
+    // Get pre-defined tokens
+    Map<String, String> globalTokens =
+        instance.getComponent().getScheduler().globalTokens;
+    Map<String, String> tokensForSubstitution = ProviderUtils
+        .initCompTokensForSubstitute(instance);
+    tokensForSubstitution.putAll(globalTokens);
+    // Set the environment variables in launcher
+    launcher.putEnv(SliderUtils
+        .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+    launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+    launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    if (System.getenv(HADOOP_USER_NAME) != null) {
+      launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+    }
+    launcher.setEnv("LANG", "en_US.UTF-8");
+    launcher.setEnv("LC_ALL", "en_US.UTF-8");
+    launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+    for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+      tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+    }
+    //TODO add component host tokens?
+//    ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+    // create config file on hdfs and add local resource
+    ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+        component, tokensForSubstitution, instance, context);
+
+    // substitute launch command
+    String launchCommand = ProviderUtils
+        .substituteStrWithTokens(component.getLaunchCommand(),
+            tokensForSubstitution);
+    CommandLineBuilder operation = new CommandLineBuilder();
+    operation.add(launchCommand);
+    operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+    launcher.addCommand(operation.build());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
new file mode 100644
index 0000000..b53652a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for factories.
+ */
+public abstract class ProviderFactory {
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(ProviderFactory.class);
+
+  protected ProviderFactory() {}
+
+  public abstract AbstractClientProvider createClientProvider();
+
+  public abstract ProviderService createServerProvider();
+
+  public static synchronized ProviderService getProviderService(Artifact
+      artifact) {
+    return createSliderProviderFactory(artifact).createServerProvider();
+  }
+
+  public static synchronized AbstractClientProvider getClientProvider(Artifact
+      artifact) {
+    return createSliderProviderFactory(artifact).createClientProvider();
+  }
+
+  /**
+   * Create a provider for a specific application
+   * @param artifact artifact
+   * @return provider factory
+   */
+  public static synchronized ProviderFactory createSliderProviderFactory(
+      Artifact artifact) {
+    if (artifact == null || artifact.getType() == null) {
+      LOG.debug("Loading service provider type default");
+      return DefaultProviderFactory.getInstance();
+    }
+    LOG.debug("Loading service provider type {}", artifact.getType());
+    switch (artifact.getType()) {
+      // TODO add handling for custom types?
+      // TODO handle application
+      case DOCKER:
+        return DockerProviderFactory.getInstance();
+      case TARBALL:
+        return TarballProviderFactory.getInstance();
+      default:
+        throw new IllegalArgumentException(String.format("Resolution error, " +
+                "%s should not be passed to createSliderProviderFactory",
+            artifact.getType()));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
new file mode 100644
index 0000000..a28c3b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+
+import java.io.IOException;
+
+public interface ProviderService {
+
+  /**
+   * Set up the entire container launch context
+   */
+  void buildContainerLaunchContext(AbstractLauncher containerLauncher,
+      Application application, ComponentInstance instance,
+      SliderFileSystem sliderFileSystem) throws IOException, SliderException;
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message