hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [46/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
Date Thu, 31 Aug 2017 05:39:42 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
deleted file mode 100644
index 1fa07ce..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
-  public static final int DEFAULT_PORT = 2181;
-
-  public static String buildConnectionString(String zkHosts, int port) {
-    String zkPort = Integer.toString(port);
-    //parse the hosts
-    String[] hostlist = zkHosts.split(",", 0);
-    String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
-    return quorum;
-  }
-
-  /**
-   * Take a quorum list and split it to (trimmed) pairs
-   * @param hostPortQuorumList list of form h1:port, h2:port2,...
-   * @return a possibly empty list of values between commas. They may not be
-   * valid hostname:port pairs
-   */
-  public static List<String> splitToPairs(String hostPortQuorumList) {
-    // split an address hot
-    String[] strings = StringUtils.getStrings(hostPortQuorumList);
-    int len = 0;
-    if (strings != null) {
-      len = strings.length;
-    }
-    List<String> tuples = new ArrayList<String>(len);
-    if (strings != null) {
-      for (String s : strings) {
-        tuples.add(s.trim());
-      }
-    }
-    return tuples;
-  }
-
-  /**
-   * Split a quorum list into a list of hostnames and ports
-   * @param hostPortQuorumList split to a list of hosts and ports
-   * @return a list of values
-   */
-  public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
-    // split an address hot
-    String[] strings = StringUtils.getStrings(hostPortQuorumList);
-    int len = 0;
-    if (strings != null) {
-      len = strings.length;
-    }
-    List<HostAndPort> list = new ArrayList<HostAndPort>(len);
-    if (strings != null) {
-      for (String s : strings) {
-        list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
-      }
-    }
-    return list;
-  }
-
-  /**
-   * Build up to a hosts only list
-   * @param hostAndPorts
-   * @return a list of the hosts only
-   */
-  public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
-    StringBuilder sb = new StringBuilder();
-    for (HostAndPort hostAndPort : hostAndPorts) {
-      sb.append(hostAndPort.getHostText()).append(",");
-    }
-    if (sb.length() > 0) {
-      sb.delete(sb.length() - 1, sb.length());
-    }
-    return sb.toString();
-  }
-
-  public static String buildQuorumEntry(HostAndPort hostAndPort,
-    int defaultPort) {
-    String s = hostAndPort.toString();
-    if (hostAndPort.hasPort()) {
-      return s;
-    } else {
-      return s + ":" + defaultPort;
-    }
-  }
-
-  /**
-   * Build a quorum list, injecting a ":defaultPort" ref if needed on
-   * any entry without one
-   * @param hostAndPorts
-   * @param defaultPort
-   * @return
-   */
-  public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
-    List<String> entries = new ArrayList<String>(hostAndPorts.size());
-    for (HostAndPort hostAndPort : hostAndPorts) {
-      entries.add(buildQuorumEntry(hostAndPort, defaultPort));
-    }
-    return SliderUtils.join(entries, ",", false);
-  }
-  
-  public static String convertToHostsOnlyList(String quorum) throws
-      BadConfigException {
-    List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
-    return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
-  }
-
-  public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
-      BadConfigException {
-    List<HostAndPort> hostAndPorts =
-        ZookeeperUtils.splitToHostsAndPorts(quorum);
-    if (hostAndPorts.isEmpty()) {
-      throw new BadConfigException("empty zookeeper quorum");
-    }
-    return hostAndPorts;
-  }
-  
-  public static int getFirstPort(String quorum, int defVal) throws
-      BadConfigException {
-    List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
-    int port = hostAndPorts.get(0).getPortOrDefault(defVal);
-    return port;
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
deleted file mode 100644
index 0a21c24..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/ClientAMProtocol.proto
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.yarn.proto";
-option java_outer_classname = "ClientAMProtocol";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.yarn;
-
-service ClientAMProtocolService {
-  rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
-  rpc getStatus(GetStatusRequestProto) returns (GetStatusResponseProto);
-  rpc stop(StopRequestProto) returns (StopResponseProto);
-}
-
-message FlexComponentsRequestProto {
-  repeated ComponentCountProto components = 1;
-}
-
-message ComponentCountProto {
-  optional string name = 1;
-  optional int64 numberOfContainers = 2;
-}
-
-message FlexComponentsResponseProto{
-}
-
-message GetStatusRequestProto {
-
-}
-message GetStatusResponseProto {
-  optional string status = 1;
-}
-
-message StopRequestProto {
-
-}
-
-message StopResponseProto {
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
deleted file mode 100644
index 4fa81ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.base.Supplier;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentState;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
-import static org.mockito.Mockito.mock;
-
-public class MockServiceAM extends ServiceMaster {
-
-  Application application;
-  // The list of containers fed by tests to be returned on
-  // AMRMClientCallBackHandler#onContainersAllocated
-  final List<Container> feedContainers =
-      Collections.synchronizedList(new LinkedList<>());
-
-  public MockServiceAM(Application application) {
-    super(application.getName());
-    this.application = application;
-  }
-
-
-  @Override
-  protected ContainerId getAMContainerId()
-      throws BadClusterStateException {
-    return ContainerId.newContainerId(ApplicationAttemptId
-        .newInstance(ApplicationId.fromString(application.getId()), 1), 1);
-  }
-
-  @Override
-  protected Path getAppDir() {
-    Path path = new Path(new Path("target", "apps"), application.getName());
-    System.out.println("Application path: " + path);
-    return path;
-  }
-
-  @Override
-  protected ServiceScheduler createServiceScheduler(ServiceContext context)
-      throws IOException, YarnException {
-    return new ServiceScheduler(context) {
-
-      @Override
-      protected YarnRegistryViewForProviders createYarnRegistryOperations(
-          ServiceContext context, RegistryOperations registryClient) {
-        return mock(YarnRegistryViewForProviders.class);
-      }
-
-      @Override
-      protected AMRMClientAsync<AMRMClient.ContainerRequest> createAMRMClient() {
-        AMRMClientImpl client1 = new AMRMClientImpl() {
-          @Override public AllocateResponse allocate(float progressIndicator)
-              throws YarnException, IOException {
-
-            AllocateResponse.AllocateResponseBuilder builder =
-                AllocateResponse.newBuilder();
-            synchronized (feedContainers) {
-              if (feedContainers.isEmpty()) {
-                System.out.println("Allocating........ no containers");
-                return builder.build();
-              } else {
-                // The AMRMClient will return containers for compoenent that are
-                // at FLEXING state
-                List<Container> allocatedContainers = new LinkedList<>();
-                Iterator<Container> itor = feedContainers.iterator();
-                while (itor.hasNext()) {
-                  Container c = itor.next();
-                  org.apache.hadoop.yarn.service.component.Component component =
-                      componentsById.get(c.getAllocationRequestId());
-                  if (component.getState() == ComponentState.FLEXING) {
-                    System.out.println("Allocated container " + c.getId());
-                    allocatedContainers.add(c);
-                    itor.remove();
-                  }
-                }
-                return builder.allocatedContainers(allocatedContainers).build();
-              }
-            }
-          }
-
-          @Override
-          public RegisterApplicationMasterResponse registerApplicationMaster(
-              String appHostName, int appHostPort, String appTrackingUrl) {
-            return mock(RegisterApplicationMasterResponse.class);
-          }
-
-          @Override public void unregisterApplicationMaster(
-              FinalApplicationStatus appStatus, String appMessage,
-              String appTrackingUrl) {
-            // DO nothing
-          }
-        };
-
-        return AMRMClientAsync
-            .createAMRMClientAsync(client1, 1000,
-                this.new AMRMClientCallback());
-      }
-
-      @Override
-      public NMClientAsync createNMClient() {
-        NMClientAsync nmClientAsync = super.createNMClient();
-        nmClientAsync.setClient(mock(NMClient.class));
-        return nmClientAsync;
-      }
-    };
-  }
-
-  @Override protected void loadApplicationJson(ServiceContext context,
-      SliderFileSystem fs) throws IOException {
-    context.application = application;
-  }
-
-  /**
-   *
-   * @param application The application for the component
-   * @param id The id for the container
-   * @param compName The component to which the container is fed
-   * @return
-   */
-  public Container feedContainerToComp(Application application, int id,
-      String compName) {
-    ApplicationId applicationId = ApplicationId.fromString(application.getId());
-    ContainerId containerId = ContainerId
-        .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
-    NodeId nodeId = NodeId.newInstance("localhost", 1234);
-    Container container = Container
-        .newInstance(containerId, nodeId, "localhost",
-            Resource.newInstance(100, 1), Priority.newInstance(0), null);
-
-    long allocateId =
-        context.scheduler.getAllComponents().get(compName).getAllocateId();
-    container.setAllocationRequestId(allocateId);
-    synchronized (feedContainers) {
-      feedContainers.add(container);
-    }
-    return container;
-  }
-
-  public void flexComponent(String compName, long numberOfContainers)
-      throws IOException {
-    ClientAMProtocol.ComponentCountProto componentCountProto =
-        ClientAMProtocol.ComponentCountProto.newBuilder().setName(compName)
-            .setNumberOfContainers(numberOfContainers).build();
-    ClientAMProtocol.FlexComponentsRequestProto requestProto =
-        ClientAMProtocol.FlexComponentsRequestProto.newBuilder()
-            .addComponents(componentCountProto).build();
-    context.clientAMService.flexComponents(requestProto);
-  }
-
-  public Component getComponent(String compName) {
-    return context.scheduler.getAllComponents().get(compName);
-  }
-
-  public void waitForDependenciesSatisfied(String compName)
-      throws TimeoutException, InterruptedException {
-    GenericTestUtils.waitFor(new Supplier<Boolean>() {
-      @Override public Boolean get() {
-        return context.scheduler.getAllComponents().get(compName)
-            .areDependenciesReady();
-      }
-    }, 1000, 20000);
-  }
-
-  public void waitForNumDesiredContainers(String compName,
-      int numDesiredContainers) throws TimeoutException, InterruptedException {
-    GenericTestUtils.waitFor(new Supplier<Boolean>() {
-      @Override public Boolean get() {
-        return context.scheduler.getAllComponents().get(compName)
-            .getNumDesiredInstances() == numDesiredContainers;
-      }
-    }, 1000, 20000);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
deleted file mode 100644
index 73172bf..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.codehaus.jackson.map.PropertyNamingStrategy;
-
-public class ServiceTestUtils {
-
-  public static final JsonSerDeser<Application> JSON_SER_DESER =
-      new JsonSerDeser<>(Application.class,
-          PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
-
-  // Example service definition
-  // 2 components, each of which has 2 containers.
-  protected Application createExampleApplication() {
-    Application exampleApp = new Application();
-    exampleApp.setName("example-app");
-    exampleApp.addComponent(createComponent("compa"));
-    exampleApp.addComponent(createComponent("compb"));
-    return exampleApp;
-  }
-
-  protected Component createComponent(String name) {
-    return createComponent(name, 2L, "sleep 1000");
-  }
-
-  protected Component createComponent(String name, long numContainers,
-      String command) {
-    Component comp1 = new Component();
-    comp1.setNumberOfContainers(numContainers);
-    comp1.setLaunchCommand(command);
-    comp1.setName(name);
-    Resource resource = new Resource();
-    comp1.setResource(resource);
-    resource.setMemory("128");
-    resource.setCpus(1);
-    return comp1;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
deleted file mode 100644
index 1a22875..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Resource;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
-import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
-import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Test for ServiceApiUtil helper methods.
- */
-public class TestServiceApiUtil {
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TestServiceApiUtil.class);
-  private static final String EXCEPTION_PREFIX = "Should have thrown " +
-      "exception: ";
-  private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
-      "exception: ";
-
-  private static final String LEN_64_STR =
-      "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
-
-  private static final YarnConfiguration CONF_DEFAULT_DNS = new
-      YarnConfiguration();
-  private static final YarnConfiguration CONF_DNS_ENABLED = new
-      YarnConfiguration();
-
-  @BeforeClass
-  public static void init() {
-    CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
-  }
-
-  @Test(timeout = 90000)
-  public void testResourceValidation() throws Exception {
-    assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
-        .length());
-
-    SliderFileSystem sfs = initMock(null);
-
-    Application app = new Application();
-
-    // no name
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no name");
-    } catch (IllegalArgumentException e) {
-      assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
-    }
-
-    // bad format name
-    String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
-    for (String badName : badNames) {
-      app.setName(badName);
-      try {
-        ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-        Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
-      } catch (IllegalArgumentException e) {
-        assertEquals(String.format(
-            ERROR_APPLICATION_NAME_INVALID_FORMAT, badName), e.getMessage());
-      }
-    }
-
-    // launch command not specified
-    app.setName(LEN_64_STR);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
-      Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
-    } catch (IllegalArgumentException e) {
-      assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
-          e.getMessage());
-    }
-
-    // launch command not specified
-    app.setName(LEN_64_STR.substring(0, RegistryConstants
-        .MAX_FQDN_LABEL_LENGTH));
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
-    } catch (IllegalArgumentException e) {
-      assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
-          e.getMessage());
-    }
-
-    // resource not specified
-    app.setLaunchCommand("sleep 3600");
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no resource");
-    } catch (IllegalArgumentException e) {
-      assertEquals(String.format(
-          RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
-          DEFAULT_COMPONENT_NAME), e.getMessage());
-    }
-
-    // memory not specified
-    Resource res = new Resource();
-    app.setResource(res);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no memory");
-    } catch (IllegalArgumentException e) {
-      assertEquals(String.format(
-          RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
-          DEFAULT_COMPONENT_NAME), e.getMessage());
-    }
-
-    // invalid no of cpus
-    res.setMemory("100mb");
-    res.setCpus(-2);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(
-          EXCEPTION_PREFIX + "application with invalid no of cpus");
-    } catch (IllegalArgumentException e) {
-      assertEquals(String.format(
-          RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
-          DEFAULT_COMPONENT_NAME), e.getMessage());
-    }
-
-    // number of containers not specified
-    res.setCpus(2);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no container count");
-    } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage()
-          .contains(ERROR_CONTAINERS_COUNT_INVALID));
-    }
-
-    // specifying profile along with cpus/memory raises exception
-    res.setProfile("hbase_finance_large");
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX
-          + "application with resource profile along with cpus/memory");
-    } catch (IllegalArgumentException e) {
-      assertEquals(String.format(RestApiErrorMessages
-              .ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
-          DEFAULT_COMPONENT_NAME),
-          e.getMessage());
-    }
-
-    // currently resource profile alone is not supported.
-    // TODO: remove the next test once resource profile alone is supported.
-    res.setCpus(null);
-    res.setMemory(null);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with resource profile only");
-    } catch (IllegalArgumentException e) {
-      assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
-          e.getMessage());
-    }
-
-    // unset profile here and add cpus/memory back
-    res.setProfile(null);
-    res.setCpus(2);
-    res.setMemory("2gb");
-
-    // null number of containers
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "null number of containers");
-    } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage()
-          .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
-    }
-
-    // negative number of containers
-    app.setNumberOfContainers(-1L);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
-    } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage()
-          .startsWith(ERROR_CONTAINERS_COUNT_INVALID));
-    }
-
-    // everything valid here
-    app.setNumberOfContainers(5L);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-    } catch (IllegalArgumentException e) {
-      LOG.error("application attributes specified should be valid here", e);
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-  }
-
-  @Test
-  public void testArtifacts() throws IOException {
-    SliderFileSystem sfs = initMock(null);
-
-    Application app = new Application();
-    app.setName("name");
-    Resource res = new Resource();
-    app.setResource(res);
-    res.setMemory("512M");
-    app.setNumberOfContainers(3L);
-
-    // no artifact id fails with default type
-    Artifact artifact = new Artifact();
-    app.setArtifact(artifact);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
-    } catch (IllegalArgumentException e) {
-      assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
-    }
-
-    // no artifact id fails with APPLICATION type
-    artifact.setType(Artifact.TypeEnum.APPLICATION);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
-    } catch (IllegalArgumentException e) {
-      assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
-    }
-
-    // no artifact id fails with TARBALL type
-    artifact.setType(Artifact.TypeEnum.TARBALL);
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
-    } catch (IllegalArgumentException e) {
-      assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
-    }
-
-    // everything valid here
-    artifact.setType(Artifact.TypeEnum.DOCKER);
-    artifact.setId("docker.io/centos:centos7");
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-    } catch (IllegalArgumentException e) {
-      LOG.error("application attributes specified should be valid here", e);
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-
-    // defaults assigned
-    assertEquals(app.getComponents().get(0).getName(),
-        DEFAULT_COMPONENT_NAME);
-    assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
-  }
-
-  private static Resource createValidResource() {
-    Resource res = new Resource();
-    res.setMemory("512M");
-    return res;
-  }
-
-  private static Component createValidComponent(String compName) {
-    Component comp = new Component();
-    comp.setName(compName);
-    comp.setResource(createValidResource());
-    comp.setNumberOfContainers(1L);
-    return comp;
-  }
-
-  private static Application createValidApplication(String compName) {
-    Application app = new Application();
-    app.setLaunchCommand("sleep 3600");
-    app.setName("name");
-    app.setResource(createValidResource());
-    app.setNumberOfContainers(1L);
-    if (compName != null) {
-      app.addComponent(createValidComponent(compName));
-    }
-    return app;
-  }
-
-  private static SliderFileSystem initMock(Application ext) throws IOException {
-    SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-    FileSystem mockFs = createNiceMock(FileSystem.class);
-    JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
-        .class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    if (ext != null) {
-      expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
-          .anyTimes();
-    }
-    replay(sfs, mockFs, jsonSerDeser);
-    ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
-    return sfs;
-  }
-
-  @Test
-  public void testExternalApplication() throws IOException {
-    Application ext = createValidApplication("comp1");
-    SliderFileSystem sfs = initMock(ext);
-
-    Application app = createValidApplication(null);
-
-    Artifact artifact = new Artifact();
-    artifact.setType(Artifact.TypeEnum.APPLICATION);
-    artifact.setId("id");
-    app.setArtifact(artifact);
-
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-    } catch (IllegalArgumentException e) {
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-
-    assertEquals(1, app.getComponents().size());
-    assertNotNull(app.getComponent("comp1"));
-  }
-
-  @Test
-  public void testDuplicateComponents() throws IOException {
-    SliderFileSystem sfs = initMock(null);
-
-    String compName = "comp1";
-    Application app = createValidApplication(compName);
-    app.addComponent(createValidComponent(compName));
-
-    // duplicate component name fails
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with component collision");
-    } catch (IllegalArgumentException e) {
-      assertEquals("Component name collision: " + compName, e.getMessage());
-    }
-  }
-
-  @Test
-  public void testExternalDuplicateComponent() throws IOException {
-    Application ext = createValidApplication("comp1");
-    SliderFileSystem sfs = initMock(ext);
-
-    Application app = createValidApplication("comp1");
-    Artifact artifact = new Artifact();
-    artifact.setType(Artifact.TypeEnum.APPLICATION);
-    artifact.setId("id");
-    app.getComponent("comp1").setArtifact(artifact);
-
-    // duplicate component name okay in the case of APPLICATION component
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-    } catch (IllegalArgumentException e) {
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-  }
-
-  @Test
-  public void testExternalComponent() throws IOException {
-    Application ext = createValidApplication("comp1");
-    SliderFileSystem sfs = initMock(ext);
-
-    Application app = createValidApplication("comp2");
-    Artifact artifact = new Artifact();
-    artifact.setType(Artifact.TypeEnum.APPLICATION);
-    artifact.setId("id");
-    app.setArtifact(artifact);
-
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-    } catch (IllegalArgumentException e) {
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-
-    assertEquals(1, app.getComponents().size());
-    // artifact ID not inherited from global
-    assertNotNull(app.getComponent("comp2"));
-
-    // set APPLICATION artifact id on component
-    app.getComponent("comp2").setArtifact(artifact);
-
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-    } catch (IllegalArgumentException e) {
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-
-    assertEquals(1, app.getComponents().size());
-    // original component replaced by external component
-    assertNotNull(app.getComponent("comp1"));
-  }
-
-  public static void verifyDependencySorting(List<Component> components,
-      Component... expectedSorting) {
-    Collection<Component> actualSorting = ServiceApiUtil.sortByDependencies(
-        components);
-    assertEquals(expectedSorting.length, actualSorting.size());
-    int i = 0;
-    for (Component component : actualSorting) {
-      assertEquals(expectedSorting[i++], component);
-    }
-  }
-
-  @Test
-  public void testDependencySorting() throws IOException {
-    Component a = new Component().name("a");
-    Component b = new Component().name("b");
-    Component c = new Component().name("c");
-    Component d = new Component().name("d").dependencies(Arrays.asList("c"));
-    Component e = new Component().name("e").dependencies(Arrays.asList("b",
-        "d"));
-
-    verifyDependencySorting(Arrays.asList(a, b, c), a, b, c);
-    verifyDependencySorting(Arrays.asList(c, a, b), c, a, b);
-    verifyDependencySorting(Arrays.asList(a, b, c, d, e), a, b, c, d, e);
-    verifyDependencySorting(Arrays.asList(e, d, c, b, a), c, b, a, d, e);
-
-    c.setDependencies(Arrays.asList("e"));
-    try {
-      verifyDependencySorting(Arrays.asList(a, b, c, d, e));
-      Assert.fail(EXCEPTION_PREFIX + "components with dependency cycle");
-    } catch (IllegalArgumentException ex) {
-      assertEquals(String.format(
-          RestApiErrorMessages.ERROR_DEPENDENCY_CYCLE, Arrays.asList(c, d,
-              e)), ex.getMessage());
-    }
-
-    SliderFileSystem sfs = initMock(null);
-    Application application = createValidApplication(null);
-    application.setComponents(Arrays.asList(c, d, e));
-    try {
-      ServiceApiUtil.validateAndResolveApplication(application, sfs,
-          CONF_DEFAULT_DNS);
-      Assert.fail(EXCEPTION_PREFIX + "components with bad dependencies");
-    } catch (IllegalArgumentException ex) {
-      assertEquals(String.format(
-          RestApiErrorMessages.ERROR_DEPENDENCY_INVALID, "b", "e"), ex
-          .getMessage());
-    }
-  }
-
-  @Test
-  public void testInvalidComponent() throws IOException {
-    SliderFileSystem sfs = initMock(null);
-    testComponent(sfs);
-  }
-
-  @Test
-  public void testValidateCompName() {
-    String[] invalidNames = {
-        "EXAMPLE", // UPPER case not allowed
-        "example_app" // underscore not allowed.
-    };
-    for (String name : invalidNames) {
-      try {
-        ServiceApiUtil.validateCompName(name);
-        Assert.fail();
-      } catch (IllegalArgumentException ex) {
-        ex.printStackTrace();
-      }
-    }
-  }
-
-  private static void testComponent(SliderFileSystem sfs)
-      throws IOException {
-    int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
-    assertEquals(19, Long.toString(Long.MAX_VALUE).length());
-    maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
-
-    String compName = LEN_64_STR.substring(0, maxLen + 1);
-    Application app = createValidApplication(null);
-    app.addComponent(createValidComponent(compName));
-
-    // invalid component name fails if dns is enabled
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-      Assert.fail(EXCEPTION_PREFIX + "application with invalid component name");
-    } catch (IllegalArgumentException e) {
-      assertEquals(String.format(RestApiErrorMessages
-          .ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
-    }
-
-    // does not fail if dns is disabled
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
-    } catch (IllegalArgumentException e) {
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-
-    compName = LEN_64_STR.substring(0, maxLen);
-    app = createValidApplication(null);
-    app.addComponent(createValidComponent(compName));
-
-    // does not fail
-    try {
-      ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
-    } catch (IllegalArgumentException e) {
-      Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
deleted file mode 100644
index a36e0b4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.curator.test.TestingCluster;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
-import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.AM_RESOURCE_MEM;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * End to end tests to test deploying services with MiniYarnCluster and a in-JVM
- * ZK testing cluster.
- */
-public class TestYarnNativeServices extends ServiceTestUtils{
-
-  private static final Log LOG =
-      LogFactory.getLog(TestYarnNativeServices.class);
-
-  private MiniYARNCluster yarnCluster = null;
-  private MiniDFSCluster hdfsCluster = null;
-  private FileSystem fs = null;
-  protected Configuration conf = null;
-  private static final int NUM_NMS = 1;
-  private File basedir;
-
-  @Rule
-  public TemporaryFolder tmpFolder = new TemporaryFolder();
-
-  @Before
-  public void setup() throws Exception {
-    setupInternal(NUM_NMS);
-  }
-
-  private void setupInternal(int numNodeManager)
-      throws Exception {
-    LOG.info("Starting up YARN cluster");
-//    Logger rootLogger = LogManager.getRootLogger();
-//    rootLogger.setLevel(Level.DEBUG);
-    conf = new YarnConfiguration();
-    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
-    // reduce the teardown waiting time
-    conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);
-    conf.set("yarn.log.dir", "target");
-    // mark if we need to launch the v1 timeline server
-    // disable aux-service based timeline aggregators
-    conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
-    conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
-    // Enable ContainersMonitorImpl
-    conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
-        LinuxResourceCalculatorPlugin.class.getName());
-    conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
-        ProcfsBasedProcessTree.class.getName());
-    conf.setBoolean(
-        YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, true);
-    conf.setBoolean(TIMELINE_SERVICE_ENABLED, false);
-    conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
-    conf.setLong(DEBUG_NM_DELETE_DELAY_SEC, 60000);
-    conf.setLong(AM_RESOURCE_MEM, 526);
-    conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 5);
-    // Disable vmem check to disallow NM killing the container
-    conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
-    conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
-    // setup zk cluster
-    TestingCluster zkCluster;
-    zkCluster = new TestingCluster(1);
-    zkCluster.start();
-    conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
-    conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
-    LOG.info("ZK cluster: " +  zkCluster.getConnectString());
-
-    fs = FileSystem.get(conf);
-    basedir = new File("target", "apps");
-    if (basedir.exists()) {
-      FileUtils.deleteDirectory(basedir);
-    } else {
-      basedir.mkdirs();
-    }
-
-    conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-
-    if (yarnCluster == null) {
-      yarnCluster =
-          new MiniYARNCluster(TestYarnNativeServices.class.getSimpleName(), 1,
-              numNodeManager, 1, 1);
-      yarnCluster.init(conf);
-      yarnCluster.start();
-
-      waitForNMsToRegister();
-
-      URL url = Thread.currentThread().getContextClassLoader()
-          .getResource("yarn-site.xml");
-      if (url == null) {
-        throw new RuntimeException(
-            "Could not find 'yarn-site.xml' dummy file in classpath");
-      }
-      Configuration yarnClusterConfig = yarnCluster.getConfig();
-      yarnClusterConfig.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
-          new File(url.getPath()).getParent());
-      //write the document to a buffer (not directly to the file, as that
-      //can cause the file being written to get read -which will then fail.
-      ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
-      yarnClusterConfig.writeXml(bytesOut);
-      bytesOut.close();
-      //write the bytes to the file in the classpath
-      OutputStream os = new FileOutputStream(new File(url.getPath()));
-      os.write(bytesOut.toByteArray());
-      os.close();
-      LOG.info("Write yarn-site.xml configs to: " + url);
-    }
-    if (hdfsCluster == null) {
-      HdfsConfiguration hdfsConfig = new HdfsConfiguration();
-      hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
-          .numDataNodes(1).build();
-    }
-
-    try {
-      Thread.sleep(2000);
-    } catch (InterruptedException e) {
-      LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
-    }
-
-
-  }
-
-  private void waitForNMsToRegister() throws Exception {
-    int sec = 60;
-    while (sec >= 0) {
-      if (yarnCluster.getResourceManager().getRMContext().getRMNodes().size()
-          >= NUM_NMS) {
-        break;
-      }
-      Thread.sleep(1000);
-      sec--;
-    }
-  }
-
-  @After
-  public void tearDown() throws IOException {
-    if (yarnCluster != null) {
-      try {
-        yarnCluster.stop();
-      } finally {
-        yarnCluster = null;
-      }
-    }
-    if (hdfsCluster != null) {
-      try {
-        hdfsCluster.shutdown();
-      } finally {
-        hdfsCluster = null;
-      }
-    }
-    if (basedir != null) {
-      FileUtils.deleteDirectory(basedir);
-    }
-    SliderFileSystem sfs = new SliderFileSystem(conf);
-    Path appDir = sfs.getBaseApplicationPath();
-    sfs.getFileSystem().delete(appDir, true);
-  }
-
-
-
-  // End-to-end test to use ServiceClient to deploy a service.
-  // 1. Create a service with 2 components, each of which has 2 containers
-  // 2. Flex up each component to 3 containers and check the component instance names
-  // 3. Flex down each component to 1 container and check the component instance names
-  // 4. Flex up each component to 2 containers and check the component instance names
-  // 5. Stop the service
-  // 6. Destroy the service
-  @Test (timeout = 200000)
-  public void testCreateFlexStopDestroyService() throws Exception {
-    ServiceClient client = createClient();
-    Application exampleApp = createExampleApplication();
-    client.actionCreate(exampleApp);
-    SliderFileSystem fileSystem = new SliderFileSystem(conf);
-    Path appDir = fileSystem.buildClusterDirPath(exampleApp.getName());
-    // check app.json is persisted.
-    Assert.assertTrue(
-        fs.exists(new Path(appDir, exampleApp.getName() + ".json")));
-    waitForAllCompToBeReady(client, exampleApp);
-
-    // Flex two components, each from 2 container to 3 containers.
-    flexComponents(client, exampleApp, 3L);
-    // wait for flex to be completed, increase from 2 to 3 containers.
-    waitForAllCompToBeReady(client, exampleApp);
-    // check all instances name for each component are in sequential order.
-    checkCompInstancesInOrder(client, exampleApp);
-
-    // flex down to 1
-    flexComponents(client, exampleApp, 1L);
-    waitForAllCompToBeReady(client, exampleApp);
-    checkCompInstancesInOrder(client, exampleApp);
-
-    // check component dir and registry are cleaned up.
-
-    // flex up again to 2
-    flexComponents(client, exampleApp, 2L);
-    waitForAllCompToBeReady(client, exampleApp);
-    checkCompInstancesInOrder(client, exampleApp);
-
-    // stop the service
-    LOG.info("Stop the service");
-    client.actionStop(exampleApp.getName(), true);
-    ApplicationReport report = client.getYarnClient()
-        .getApplicationReport(ApplicationId.fromString(exampleApp.getId()));
-    // AM unregisters with RM successfully
-    Assert.assertEquals(FINISHED, report.getYarnApplicationState());
-    Assert.assertEquals(FinalApplicationStatus.ENDED,
-        report.getFinalApplicationStatus());
-
-    LOG.info("Destroy the service");
-    //destroy the service and check the app dir is deleted from fs.
-    client.actionDestroy(exampleApp.getName());
-    // check the application dir on hdfs (in this case, local fs) are deleted.
-    Assert.assertFalse(fs.exists(appDir));
-  }
-
-  // Create compa with 2 containers
-  // Create compb with 2 containers which depends on compa
-  // Check containers for compa started before containers for compb
-  @Test (timeout = 200000)
-  public void testComponentStartOrder() throws Exception {
-    ServiceClient client = createClient();
-    Application exampleApp = new Application();
-    exampleApp.setName("teststartorder");
-    exampleApp.addComponent(createComponent("compa", 2, "sleep 1000"));
-    Component compb = createComponent("compb", 2, "sleep 1000");
-
-    // Let compb depedends on compa;
-    compb.setDependencies(Collections.singletonList("compa"));
-    exampleApp.addComponent(compb);
-
-    client.actionCreate(exampleApp);
-    waitForAllCompToBeReady(client, exampleApp);
-
-    // check that containers for compa are launched before containers for compb
-    checkContainerLaunchDependencies(client, exampleApp, "compa", "compb");
-
-    client.actionStop(exampleApp.getName(), true);
-    client.actionDestroy(exampleApp.getName());
-  }
-
-  // Check containers launched are in dependency order
-  // Get all containers into a list and sort based on container launch time e.g.
-  // compa-c1, compa-c2, compb-c1, compb-c2;
-  // check that the container's launch time are align with the dependencies.
-  private void checkContainerLaunchDependencies(ServiceClient client,
-      Application exampleApp, String... compOrder)
-      throws IOException, YarnException {
-    Application retrievedApp = client.getStatus(exampleApp.getName());
-    List<Container> containerList = new ArrayList<>();
-    for (Component component : retrievedApp.getComponents()) {
-      containerList.addAll(component.getContainers());
-    }
-    // sort based on launchTime
-    containerList
-        .sort((o1, o2) -> o1.getLaunchTime().compareTo(o2.getLaunchTime()));
-    LOG.info("containerList: " + containerList);
-    // check the containers are in the dependency order.
-    int index = 0;
-    for (String comp : compOrder) {
-      long num = retrievedApp.getComponent(comp).getNumberOfContainers();
-      for (int i = 0; i < num; i++) {
-        String compInstanceName = containerList.get(index).getComponentName();
-        String compName =
-            compInstanceName.substring(0, compInstanceName.lastIndexOf('-'));
-        Assert.assertEquals(comp, compName);
-        index++;
-      }
-    }
-  }
-
-
-  private Map<String, Long> flexComponents(ServiceClient client,
-      Application exampleApp, long count) throws YarnException, IOException {
-    Map<String, Long> compCounts = new HashMap<>();
-    compCounts.put("compa", count);
-    compCounts.put("compb", count);
-    // flex will update the persisted conf to reflect latest number of containers.
-    exampleApp.getComponent("compa").setNumberOfContainers(count);
-    exampleApp.getComponent("compb").setNumberOfContainers(count);
-    client.flexByRestService(exampleApp.getName(), compCounts);
-    return compCounts;
-  }
-
-  // Check each component's comp instances name are in sequential order.
-  // E.g. If there are two instances compA-1 and compA-2
-  // When flex up to 4 instances, it should be compA-1 , compA-2, compA-3, compA-4
-  // When flex down to 3 instances,  it should be compA-1 , compA-2, compA-3.
-  private void checkCompInstancesInOrder(ServiceClient client,
-      Application exampleApp) throws IOException, YarnException {
-    Application application = client.getStatus(exampleApp.getName());
-    for (Component comp : application.getComponents()) {
-      checkEachCompInstancesInOrder(comp);
-    }
-  }
-
-  private void checkRegistryAndCompDirDeleted() {
-
-  }
-
-  private void checkEachCompInstancesInOrder(Component component) {
-    long expectedNumInstances = component.getNumberOfContainers();
-    Assert.assertEquals(expectedNumInstances, component.getContainers().size());
-    TreeSet<String> instances = new TreeSet<>();
-    for (Container container : component.getContainers()) {
-      instances.add(container.getComponentName());
-    }
-
-    int i = 0;
-    for (String s : instances) {
-      Assert.assertEquals(component.getName() + "-" + i, s);
-      i++;
-    }
-  }
-
-  private void waitForOneCompToBeReady(ServiceClient client,
-      Application exampleApp, String readyComp)
-      throws TimeoutException, InterruptedException {
-    long numExpectedContainers =
-        exampleApp.getComponent(readyComp).getNumberOfContainers();
-    GenericTestUtils.waitFor(() -> {
-      try {
-        Application retrievedApp = client.getStatus(exampleApp.getName());
-        Component retrievedComp = retrievedApp.getComponent(readyComp);
-
-        if (retrievedComp.getContainers() != null
-            && retrievedComp.getContainers().size() == numExpectedContainers) {
-          LOG.info(readyComp + " found " + numExpectedContainers
-              + " containers running");
-          return true;
-        } else {
-          LOG.info(" Waiting for " + readyComp + "'s containers to be running");
-          return false;
-        }
-      } catch (Exception e) {
-        e.printStackTrace();
-        return false;
-      }
-    }, 2000, 200000);
-  }
-
-  // wait until all the containers for all components become ready state
-  private void waitForAllCompToBeReady(ServiceClient client,
-      Application exampleApp) throws TimeoutException, InterruptedException {
-    int expectedTotalContainers = countTotalContainers(exampleApp);
-    GenericTestUtils.waitFor(() -> {
-      try {
-        Application retrievedApp = client.getStatus(exampleApp.getName());
-        int totalReadyContainers = 0;
-        LOG.info("Num Components " + retrievedApp.getComponents().size());
-        for (Component component : retrievedApp.getComponents()) {
-          LOG.info("looking for  " + component.getName());
-          LOG.info(component);
-          if (component.getContainers() != null) {
-            if (component.getContainers().size() == exampleApp
-                .getComponent(component.getName()).getNumberOfContainers()) {
-              for (Container container : component.getContainers()) {
-                LOG.info(
-                    "Container state " + container.getState() + ", component "
-                        + component.getName());
-                if (container.getState() == ContainerState.READY) {
-                  totalReadyContainers++;
-                  LOG.info("Found 1 ready container " + container.getId());
-                }
-              }
-            } else {
-              LOG.info(component.getName() + " Expected number of containers "
-                  + exampleApp.getComponent(component.getName())
-                  .getNumberOfContainers() + ", current = " + component
-                  .getContainers());
-            }
-          }
-        }
-        LOG.info("Exit loop, totalReadyContainers= " + totalReadyContainers
-            + " expected = " + expectedTotalContainers);
-        return totalReadyContainers == expectedTotalContainers;
-      } catch (Exception e) {
-        e.printStackTrace();
-        return false;
-      }
-    }, 2000, 200000);
-  }
-
-  private ServiceClient createClient() throws Exception {
-    ServiceClient client = new ServiceClient() {
-      @Override protected Path addJarResource(String appName,
-          Map<String, LocalResource> localResources)
-          throws IOException, SliderException {
-        // do nothing, the Unit test will use local jars
-        return null;
-      }
-    };
-    client.init(conf);
-    client.start();
-    return client;
-  }
-
-
-  private int countTotalContainers(Application application) {
-    int totalContainers = 0;
-    for (Component component : application.getComponents()) {
-      totalContainers += component.getNumberOfContainers();
-    }
-    return totalContainers;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
deleted file mode 100644
index a22c000..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-
-/**
- * Test for building / resolving components of type APPLICATION.
- */
-public class TestBuildExternalComponents {
-
-  protected Configuration conf = new YarnConfiguration();
-  private File basedir;
-
-  // Check component names match with expected
-  private static void checkComponentNames(List<Component> components,
-      Set<String> expectedComponents) {
-    Assert.assertEquals(expectedComponents.size(), components.size());
-    for (Component comp : components) {
-      Assert.assertTrue(expectedComponents.contains(comp.getName()));
-    }
-  }
-
-  // 1. Build the appDef and store on fs
-  // 2. check component names
-  private void buildAndCheckComponents(String appName, String appDef,
-      SliderFileSystem sfs, Set<String> names) throws Throwable {
-    String[] args =
-        { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
-    ClientArgs clientArgs = new ClientArgs(args);
-    clientArgs.parse();
-    ServiceCLI cli = new ServiceCLI() {
-      @Override protected void createServiceClient() {
-        client = new ServiceClient();
-        client.init(conf);
-        client.start();
-      }
-    };
-    cli.exec(clientArgs);
-
-    // verify generated conf
-    List<Component> components =
-        ServiceApiUtil.getApplicationComponents(sfs, appName);
-    checkComponentNames(components, names);
-  }
-
-  @Before
-  public void setup() throws IOException {
-    basedir = new File("target", "apps");
-    if (basedir.exists()) {
-      FileUtils.deleteDirectory(basedir);
-    } else {
-      basedir.mkdirs();
-    }
-    conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-  }
-
-  @After
-  public void tearDown() throws IOException {
-    if (basedir != null) {
-      FileUtils.deleteDirectory(basedir);
-    }
-  }
-
-  // Test applications defining external components(APPLICATION type)
-  // can be resolved correctly
-  @Test
-  public void testExternalComponentBuild() throws Throwable {
-    SliderFileSystem sfs = new SliderFileSystem(conf);
-
-    Set<String> nameSet = new HashSet<>();
-    nameSet.add("simple");
-    nameSet.add("master");
-    nameSet.add("worker");
-
-    // app-1 has 3 components: simple, master, worker
-    buildAndCheckComponents("app-1", ExampleAppJson.APP_JSON, sfs, nameSet);
-    buildAndCheckComponents("external-0", ExampleAppJson.EXTERNAL_JSON_0, sfs,
-        nameSet);
-
-    nameSet.add("other");
-
-    // external1 has 3 components: simple(APPLICATION - app1), master and other
-    buildAndCheckComponents("external-1", ExampleAppJson.EXTERNAL_JSON_1, sfs,
-        nameSet);
-
-    nameSet.add("another");
-
-    // external2 has 2 components: ext(APPLICATION - external1), another
-    buildAndCheckComponents("external-2", ExampleAppJson.EXTERNAL_JSON_2, sfs,
-        nameSet);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
deleted file mode 100644
index 20c06ab..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_APPDEF;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.mock;
-
-public class TestServiceCLI {
-
-  protected Configuration conf = new YarnConfiguration();
-  private File basedir;
-  private ServiceCLI cli;
-  private SliderFileSystem fs;
-
-  private void buildApp(String appName, String appDef) throws Throwable {
-    String[] args =
-        { "build", appName, ARG_APPDEF, ExampleAppJson.resourceName(appDef) };
-    ClientArgs clientArgs = new ClientArgs(args);
-    clientArgs.parse();
-    cli.exec(clientArgs);
-  }
-
-  @Before
-  public void setup() throws Throwable {
-    basedir = new File("target", "apps");
-    conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-    fs = new SliderFileSystem(conf);
-    if (basedir.exists()) {
-      FileUtils.deleteDirectory(basedir);
-    } else {
-      basedir.mkdirs();
-    }
-
-    // create a CLI and skip connection to AM
-    cli = new ServiceCLI() {
-      @Override protected void createServiceClient() {
-        client = new ServiceClient() {
-          @Override protected ClientAMProtocol getAMProxy(String appName,
-              ApplicationReport report) throws IOException {
-            return mock(ClientAMProtocol.class);
-          }
-          @Override protected ClientAMProtocol getAMProxy(String appName)
-              throws IOException, YarnException {
-            return mock(ClientAMProtocol.class);
-          }
-        };
-        client.init(conf);
-        client.start();
-      }
-    };
-  }
-
-  @After
-  public void tearDown() throws IOException {
-    if (basedir != null) {
-      FileUtils.deleteDirectory(basedir);
-    }
-  }
-
-  // Test flex components count are persisted.
-  @Test
-  public void testFlexComponents() throws Throwable {
-    buildApp("service-1", ExampleAppJson.APP_JSON);
-
-    checkCompCount("master", 1L);
-
-    // increase by 2
-    String[] flexUpArgs = {"flex", "service-1", "--component", "master" , "+2"};
-    ClientArgs clientArgs = new ClientArgs(flexUpArgs);
-    clientArgs.parse();
-    cli.exec(clientArgs);
-
-    checkCompCount("master", 3L);
-
-    // decrease by 1
-    String[] flexDownArgs = {"flex", "service-1", "--component", "master", "-1"};
-    clientArgs = new ClientArgs(flexDownArgs);
-    clientArgs.parse();
-    cli.exec(clientArgs);
-
-    checkCompCount("master", 2L);
-
-    String[] flexAbsoluteArgs = {"flex", "service-1", "--component", "master", "10"};
-    clientArgs = new ClientArgs(flexAbsoluteArgs);
-    clientArgs.parse();
-    cli.exec(clientArgs);
-
-    checkCompCount("master", 10L);
-  }
-
-  private void checkCompCount(String compName, long count) throws IOException {
-    List<Component> components =
-        ServiceApiUtil.getApplicationComponents(fs, "service-1");
-    for (Component component : components) {
-      if (component.getName().equals(compName)) {
-        Assert.assertEquals(count, component.getNumberOfContainers().longValue());
-        return;
-      }
-    }
-    Assert.fail();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
deleted file mode 100644
index 9e13200..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/ExampleAppJson.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-
-import org.apache.hadoop.yarn.service.api.records.Application;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-
-/**
- * Names of the example configs.
- */
-public final class ExampleAppJson {
-
-  public static final String APP_JSON = "app.json";
-  public static final String OVERRIDE_JSON = "app-override.json";
-  public static final String DEFAULT_JSON = "default.json";
-  public static final String EXTERNAL_JSON_0 = "external0.json";
-  public static final String EXTERNAL_JSON_1 = "external1.json";
-  public static final String EXTERNAL_JSON_2 = "external2.json";
-
-  public static final String PACKAGE = "/org/apache/hadoop/yarn/service/conf/examples/";
-
-
-  private static final String[] ALL_EXAMPLES = {APP_JSON, OVERRIDE_JSON,
-      DEFAULT_JSON};
-
-  public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>();
-  static {
-    for (String example : ALL_EXAMPLES) {
-      ALL_EXAMPLE_RESOURCES.add(PACKAGE + example);
-    }
-  }
-
-  private ExampleAppJson() {
-  }
-
-  public static Application loadResource(String name) throws IOException {
-    return JSON_SER_DESER.fromResource(PACKAGE + name);
-  }
-
-  public static String resourceName(String name) {
-    return "target/test-classes" + PACKAGE + name;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e94f23ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
deleted file mode 100644
index 954d117..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestAppJsonResolve.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.api.records.Configuration;
-import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.hadoop.yarn.service.conf.ExampleAppJson.*;
-import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test global configuration resolution.
- */
-public class TestAppJsonResolve extends Assert {
-  protected static final Logger LOG =
-      LoggerFactory.getLogger(TestAppJsonResolve.class);
-
-  @Test
-  public void testOverride() throws Throwable {
-    Application orig = ExampleAppJson.loadResource(OVERRIDE_JSON);
-
-    Configuration global = orig.getConfiguration();
-    assertEquals("a", global.getProperty("g1"));
-    assertEquals("b", global.getProperty("g2"));
-    assertEquals(2, global.getFiles().size());
-
-    Configuration simple = orig.getComponent("simple").getConfiguration();
-    assertEquals(0, simple.getProperties().size());
-    assertEquals(1, simple.getFiles().size());
-
-    Configuration master = orig.getComponent("master").getConfiguration();
-    assertEquals("m", master.getProperty("name"));
-    assertEquals("overridden", master.getProperty("g1"));
-    assertEquals(0, master.getFiles().size());
-
-    Configuration worker = orig.getComponent("worker").getConfiguration();
-    LOG.info("worker = {}", worker);
-    assertEquals(3, worker.getProperties().size());
-    assertEquals(0, worker.getFiles().size());
-
-    assertEquals("worker", worker.getProperty("name"));
-    assertEquals("overridden-by-worker", worker.getProperty("g1"));
-    assertNull(worker.getProperty("g2"));
-    assertEquals("1000", worker.getProperty("timeout"));
-
-    // here is the resolution
-    SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-    FileSystem mockFs = createNiceMock(FileSystem.class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    replay(sfs, mockFs);
-    ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
-        YarnConfiguration());
-
-    global = orig.getConfiguration();
-    LOG.info("global = {}", global);
-    assertEquals("a", global.getProperty("g1"));
-    assertEquals("b", global.getProperty("g2"));
-    assertEquals(2, global.getFiles().size());
-
-    simple = orig.getComponent("simple").getConfiguration();
-    assertEquals(2, simple.getProperties().size());
-    assertEquals("a", simple.getProperty("g1"));
-    assertEquals("b", simple.getProperty("g2"));
-    assertEquals(2, simple.getFiles().size());
-
-    Set<ConfigFile> files = new HashSet<>();
-    Map<String, String> props = new HashMap<>();
-    props.put("k1", "overridden");
-    props.put("k2", "v2");
-    files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
-        .PROPERTIES).props(props));
-    files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
-        .XML).props(Collections.singletonMap("k3", "v3")));
-    assertTrue(files.contains(simple.getFiles().get(0)));
-    assertTrue(files.contains(simple.getFiles().get(1)));
-
-    master = orig.getComponent("master").getConfiguration();
-    LOG.info("master = {}", master);
-    assertEquals(3, master.getProperties().size());
-    assertEquals("m", master.getProperty("name"));
-    assertEquals("overridden", master.getProperty("g1"));
-    assertEquals("b", master.getProperty("g2"));
-    assertEquals(2, master.getFiles().size());
-
-    props.put("k1", "v1");
-    files.clear();
-    files.add(new ConfigFile().destFile("file1").type(ConfigFile.TypeEnum
-        .PROPERTIES).props(props));
-    files.add(new ConfigFile().destFile("file2").type(ConfigFile.TypeEnum
-        .XML).props(Collections.singletonMap("k3", "v3")));
-
-    assertTrue(files.contains(master.getFiles().get(0)));
-    assertTrue(files.contains(master.getFiles().get(1)));
-
-    worker = orig.getComponent("worker").getConfiguration();
-    LOG.info("worker = {}", worker);
-    assertEquals(4, worker.getProperties().size());
-
-    assertEquals("worker", worker.getProperty("name"));
-    assertEquals("overridden-by-worker", worker.getProperty("g1"));
-    assertEquals("b", worker.getProperty("g2"));
-    assertEquals("1000", worker.getProperty("timeout"));
-    assertEquals(2, worker.getFiles().size());
-
-    assertTrue(files.contains(worker.getFiles().get(0)));
-    assertTrue(files.contains(worker.getFiles().get(1)));
-  }
-
-  @Test
-  public void testOverrideExternalConfiguration() throws IOException {
-    Application orig = ExampleAppJson.loadResource(EXTERNAL_JSON_1);
-
-    Configuration global = orig.getConfiguration();
-    assertEquals(0, global.getProperties().size());
-
-    assertEquals(3, orig.getComponents().size());
-
-    Configuration simple = orig.getComponent("simple").getConfiguration();
-    assertEquals(0, simple.getProperties().size());
-
-    Configuration master = orig.getComponent("master").getConfiguration();
-    assertEquals(1, master.getProperties().size());
-    assertEquals("is-overridden", master.getProperty("g3"));
-
-    Configuration other = orig.getComponent("other").getConfiguration();
-    assertEquals(0, other.getProperties().size());
-
-    // load the external application
-    SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-    FileSystem mockFs = createNiceMock(FileSystem.class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    replay(sfs, mockFs);
-    Application ext = ExampleAppJson.loadResource(APP_JSON);
-    ServiceApiUtil.validateAndResolveApplication(ext, sfs, new
-        YarnConfiguration());
-    reset(sfs, mockFs);
-
-    // perform the resolution on original application
-    JsonSerDeser<Application> jsonSerDeser = createNiceMock(JsonSerDeser
-        .class);
-    expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-    expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-        new Path("cluster_dir_path")).anyTimes();
-    expect(jsonSerDeser.load(anyObject(), anyObject())).andReturn(ext)
-        .anyTimes();
-    replay(sfs, mockFs, jsonSerDeser);
-    ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
-    ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
-        YarnConfiguration());
-
-    global = orig.getConfiguration();
-    assertEquals(0, global.getProperties().size());
-
-    assertEquals(4, orig.getComponents().size());
-
-    simple = orig.getComponent("simple").getConfiguration();
-    assertEquals(3, simple.getProperties().size());
-    assertEquals("a", simple.getProperty("g1"));
-    assertEquals("b", simple.getProperty("g2"));
-    assertEquals("60",
-        simple.getProperty("yarn.service.failure-count-reset.window"));
-
-    master = orig.getComponent("master").getConfiguration();
-    assertEquals(5, master.getProperties().size());
-    assertEquals("512M", master.getProperty("jvm.heapsize"));
-    assertEquals("overridden", master.getProperty("g1"));
-    assertEquals("b", master.getProperty("g2"));
-    assertEquals("is-overridden", master.getProperty("g3"));
-    assertEquals("60",
-        simple.getProperty("yarn.service.failure-count-reset.window"));
-
-    Configuration worker = orig.getComponent("worker").getConfiguration();
-    LOG.info("worker = {}", worker);
-    assertEquals(4, worker.getProperties().size());
-    assertEquals("512M", worker.getProperty("jvm.heapsize"));
-    assertEquals("overridden-by-worker", worker.getProperty("g1"));
-    assertEquals("b", worker.getProperty("g2"));
-    assertEquals("60",
-        worker.getProperty("yarn.service.failure-count-reset.window"));
-
-    other = orig.getComponent("other").getConfiguration();
-    assertEquals(0, other.getProperties().size());
-  }
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message