hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [30/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He
Date Tue, 18 Apr 2017 06:08:04 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
new file mode 100644
index 0000000..7e8cf5b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.util;
+
+public interface RestApiConstants {
+  String CONTEXT_ROOT = "/services/v1";
+  String APPLICATIONS_API_RESOURCE_PATH = "/applications";
+  String CONTAINERS_API_RESOURCE_PATH = "/containers";
+  String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster";
+  String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0";
+  String METAINFO_SCHEMA_VERSION = "2.1";
+  String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker";
+
+  String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd";
+  String DEFAULT_COMPONENT_NAME = "DEFAULT";
+  String DEFAULT_IMAGE = "centos:centos6";
+  String DEFAULT_NETWORK = "bridge";
+  String DEFAULT_COMMAND_PATH = "/usr/bin/docker";
+  String DEFAULT_USE_NETWORK_SCRIPT = "yes";
+
+  String PLACEHOLDER_APP_NAME = "${APP_NAME}";
+  String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}";
+  String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}";
+
+  String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+  String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+  String PROPERTY_APP_LIFETIME = "docker.lifetime";
+  String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER";
+  Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+  Integer HTTP_STATUS_CODE_ACCEPTED = 202;
+  String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip";
+
+  Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200;
+
+  String PROPERTY_PYTHON_PATH = "python.path";
+  String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency";
+
+  String COMMAND_ORDER_SUFFIX_START = "-START";
+  String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED";
+  String EXPORT_GROUP_NAME = "QuickLinks";
+
+  Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+  Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+  Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+  Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
new file mode 100644
index 0000000..0f6247d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.util;
+
+public interface RestApiErrorMessages {
+  String ERROR_APPLICATION_NAME_INVALID =
+      "Application name is either empty or not provided";
+  String ERROR_APPLICATION_NAME_INVALID_FORMAT =
+      "Application name is not valid - only lower case letters, digits,"
+          + " underscore and hyphen are allowed";
+
+  String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
+  String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
+  String ERROR_APPLICATION_IN_USE = "Application already exists in started"
+      + " state";
+  String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in"
+      + " stopped/failed state (either restart with PUT or destroy with DELETE"
+      + " before creating a new one)";
+
+  String ERROR_SUFFIX_FOR_COMPONENT =
+      " for component %s (nor at the global level)";
+  String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
+  String ERROR_ARTIFACT_FOR_COMP_INVALID =
+      ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+  String ERROR_ARTIFACT_ID_INVALID =
+      "Artifact id (like docker image name) is either empty or not provided";
+  String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
+      ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+  String ERROR_RESOURCE_INVALID = "Resource is not provided";
+  String ERROR_RESOURCE_FOR_COMP_INVALID =
+      ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+  String ERROR_RESOURCE_MEMORY_INVALID =
+      "Application resource or memory not provided";
+  String ERROR_RESOURCE_CPUS_INVALID =
+      "Application resource or cpus not provided";
+  String ERROR_RESOURCE_CPUS_INVALID_RANGE =
+      "Unacceptable no of cpus specified, either zero or negative";
+  String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
+      ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+  String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
+      ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+  String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
+      ERROR_RESOURCE_CPUS_INVALID_RANGE
+          + " for component %s (or at the global level)";
+  String ERROR_CONTAINERS_COUNT_INVALID =
+      "Required no of containers not specified";
+  String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
+      ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+  String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
+      "Cannot specify" + " cpus/memory along with profile";
+  String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
+      ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+          + " for component %s";
+  String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
+      "Resource profile is not " + "supported yet. Please specify cpus/memory.";
+
+  String ERROR_NULL_ARTIFACT_ID =
+      "Artifact Id can not be null if artifact type is none";
+  String ERROR_ABSENT_NUM_OF_INSTANCE =
+      "Num of instances should appear either globally or per component";
+  String ERROR_ABSENT_LAUNCH_COMMAND =
+      "launch command should appear if type is slider-zip or none";
+
+  String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+      + " component level, needs corresponding values set at application level";
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
new file mode 100644
index 0000000..776ce00
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Artifact;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.Configuration;
+import org.apache.slider.api.resource.Resource;
+import org.apache.slider.common.tools.SliderUtils;
+
+public class ServiceApiUtil {
+
+  @VisibleForTesting
+  public static void validateApplicationPostPayload(Application application) {
+    if (StringUtils.isEmpty(application.getName())) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
+    }
+    if (!SliderUtils.isClusternameValid(application.getName())) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT);
+    }
+
+    // If the application has no components do top-level checks
+    if (!hasComponent(application)) {
+      // artifact
+      if (application.getArtifact() == null) {
+        throw new IllegalArgumentException(
+            RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+      }
+      if (StringUtils.isEmpty(application.getArtifact().getId())) {
+        throw new IllegalArgumentException(
+            RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+      }
+
+      // If artifact is of type APPLICATION, add a slider specific property
+      if (application.getArtifact().getType()
+          == Artifact.TypeEnum.APPLICATION) {
+        if (application.getConfiguration() == null) {
+          application.setConfiguration(new Configuration());
+        }
+      }
+      // resource
+      validateApplicationResource(application.getResource(), null,
+          application.getArtifact().getType());
+
+      // container size
+      if (application.getNumberOfContainers() == null) {
+        throw new IllegalArgumentException(
+            RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID);
+      }
+
+      // Since it is a simple app with no components, create a default component
+      application.getComponents().add(createDefaultComponent(application));
+    } else {
+      // If the application has components, then run checks for each component.
+      // Let global values take effect if component level values are not
+      // provided.
+      Artifact globalArtifact = application.getArtifact();
+      Resource globalResource = application.getResource();
+      Long globalNumberOfContainers = application.getNumberOfContainers();
+      for (Component comp : application.getComponents()) {
+        // artifact
+        if (comp.getArtifact() == null) {
+          comp.setArtifact(globalArtifact);
+        }
+        // If still null raise validation exception
+        if (comp.getArtifact() == null) {
+          throw new IllegalArgumentException(String
+              .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID,
+                  comp.getName()));
+        }
+        if (StringUtils.isEmpty(comp.getArtifact().getId())) {
+          throw new IllegalArgumentException(String
+              .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID,
+                  comp.getName()));
+        }
+
+        // If artifact is of type APPLICATION, add a slider specific property
+        if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
+          if (comp.getConfiguration() == null) {
+            comp.setConfiguration(new Configuration());
+          }
+          comp.setName(comp.getArtifact().getId());
+        }
+
+        // resource
+        if (comp.getResource() == null) {
+          comp.setResource(globalResource);
+        }
+        validateApplicationResource(comp.getResource(), comp,
+            comp.getArtifact().getType());
+
+        // container count
+        if (comp.getNumberOfContainers() == null) {
+          comp.setNumberOfContainers(globalNumberOfContainers);
+        }
+        if (comp.getNumberOfContainers() == null) {
+          throw new IllegalArgumentException(String.format(
+              RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID,
+              comp.getName()));
+        }
+      }
+    }
+
+    // Application lifetime if not specified, is set to unlimited lifetime
+    if (application.getLifetime() == null) {
+      application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME);
+    }
+  }
+
+  private static void validateApplicationResource(Resource resource,
+      Component comp, Artifact.TypeEnum artifactType) {
+    // Only apps/components of type APPLICATION can skip resource requirement
+    if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) {
+      return;
+    }
+    if (resource == null) {
+      throw new IllegalArgumentException(
+          comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String
+              .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID,
+                  comp.getName()));
+    }
+    // One and only one of profile OR cpus & memory can be specified. Specifying
+    // both raises validation error.
+    if (StringUtils.isNotEmpty(resource.getProfile()) && (
+        resource.getCpus() != null || StringUtils
+            .isNotEmpty(resource.getMemory()))) {
+      throw new IllegalArgumentException(comp == null ?
+          RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED :
+          String.format(
+              RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+              comp.getName()));
+    }
+    // Currently resource profile is not supported yet, so we will raise
+    // validation error if only resource profile is specified
+    if (StringUtils.isNotEmpty(resource.getProfile())) {
+      throw new IllegalArgumentException(
+          RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET);
+    }
+
+    String memory = resource.getMemory();
+    Integer cpus = resource.getCpus();
+    if (StringUtils.isEmpty(memory)) {
+      throw new IllegalArgumentException(
+          comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID :
+              String.format(
+                  RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID,
+                  comp.getName()));
+    }
+    if (cpus == null) {
+      throw new IllegalArgumentException(
+          comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID :
+              String.format(
+                  RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID,
+                  comp.getName()));
+    }
+    if (cpus <= 0) {
+      throw new IllegalArgumentException(comp == null ?
+          RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String
+          .format(
+              RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE,
+              comp.getName()));
+    }
+  }
+
+  public static boolean hasComponent(Application application) {
+    if (application.getComponents() == null || application.getComponents()
+        .isEmpty()) {
+      return false;
+    }
+    return true;
+  }
+
+  public static Component createDefaultComponent(Application app) {
+    Component comp = new Component();
+    comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME);
+    comp.setArtifact(app.getArtifact());
+    comp.setResource(app.getResource());
+    comp.setNumberOfContainers(app.getNumberOfContainers());
+    comp.setLaunchCommand(app.getLaunchCommand());
+    return comp;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
index b8bdc59..bfcab23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
@@ -80,22 +80,14 @@ message UpgradeContainersRequestProto {
 message UpgradeContainersResponseProto {
 }
 
-/**
- * flex the cluster
- */
-message FlexClusterRequestProto {
-  required string clusterSpec = 1;
+message FlexComponentRequestProto {
+  optional string name = 1;
+  optional int32 numberOfContainers = 2;
 }
 
-
-/**
- * flex the cluster
- */
-message FlexClusterResponseProto {
-  required bool response = 1;
+message FlexComponentResponseProto {
 }
 
-
 /**
  * void request
  */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
index d68da2b..8a0faf9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
@@ -61,24 +61,14 @@ service SliderClusterProtocolPB {
   rpc upgradeContainers(UpgradeContainersRequestProto) 
     returns(UpgradeContainersResponseProto);
 
-  /**
-   * Flex the cluster. 
-   */
-  rpc flexCluster(FlexClusterRequestProto) 
-    returns(FlexClusterResponseProto);
+  rpc flexComponent(FlexComponentRequestProto) returns (FlexComponentResponseProto);
 
   /**
    * Get the current cluster status
    */
   rpc getJSONClusterStatus(GetJSONClusterStatusRequestProto)
     returns(GetJSONClusterStatusResponseProto);
-      
-  /**
-   * Get the instance definition
-   */
-  rpc getInstanceDefinition(GetInstanceDefinitionRequestProto)
-   returns(GetInstanceDefinitionResponseProto);
-      
+
   /**
    * List all running nodes in a role
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
index a2517d5..96bfe0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
@@ -24,10 +24,6 @@
     <value>true</value>
   </property>
   <property>
-    <name>slider.provider.agent</name>
-    <value>org.apache.slider.providers.agent.AgentProviderFactory</value>
-  </property>
-  <property>
     <name>slider.provider.docker</name>
     <value>org.apache.slider.providers.docker.DockerProviderFactory</value>
   </property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
deleted file mode 100644
index b955931..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import java.lang.reflect.Method;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.LogAggregationContext;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.client.SliderYarnClientImpl;
-import org.apache.slider.common.SliderKeys;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestAppMasterLauncher {
-  SliderYarnClientImpl mockYarnClient;
-  YarnClientApplication yarnClientApp;
-  ApplicationSubmissionContext appSubmissionContext;
-  Set<String> tags = Collections.emptySet();
-  AppMasterLauncher appMasterLauncher = null;
-  boolean isOldApi = true;
-  Method rolledLogsIncludeMethod = null;
-  Method rolledLogsExcludeMethod = null;
-
-  @Before
-  public void initialize() throws Exception {
-    mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
-    yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
-    appSubmissionContext = EasyMock
-        .createNiceMock(ApplicationSubmissionContext.class);
-    EasyMock.expect(yarnClientApp.getApplicationSubmissionContext())
-        .andReturn(appSubmissionContext).once();
-    EasyMock.expect(mockYarnClient.createApplication())
-        .andReturn(yarnClientApp).once();
-
-    try {
-      LogAggregationContext.class.getMethod("newInstance", String.class,
-          String.class, String.class, String.class);
-      isOldApi = false;
-      rolledLogsIncludeMethod = LogAggregationContext.class
-          .getMethod("getRolledLogsIncludePattern");
-      rolledLogsExcludeMethod = LogAggregationContext.class
-          .getMethod("getRolledLogsExcludePattern");
-    } catch (Exception e) {
-      isOldApi = true;
-    }
-  }
-
-  /**
-   * These tests will probably fail when compiled against hadoop 2.7+. Please
-   * refer to SLIDER-810. It has been purposely not modified so that it fails
-   * and that someone needs to modify the code in
-   * {@code AbstractLauncher#extractLogAggregationContext(Map)}. Comments are
-   * provided in that method as to what needs to be done.
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testExtractLogAggregationContext() throws Exception {
-    Map<String, String> options = new HashMap<String, String>();
-    options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS,
-        " | slider*.txt  |agent.out| |");
-    options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
-        "command*.json|  agent.log*        |     ");
-
-    EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
-    appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
-        null, mockYarnClient, false, null, options, tags, null);
-
-    // Verify the include/exclude patterns
-    String expectedInclude = "slider*.txt|agent.out";
-    String expectedExclude = "command*.json|agent.log*";
-    assertPatterns(expectedInclude, expectedExclude);
-
-    EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
-
-  }
-
-  @Test
-  public void testExtractLogAggregationContextEmptyIncludePattern()
-      throws Exception {
-    Map<String, String> options = new HashMap<String, String>();
-    options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " ");
-    options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
-        "command*.json|  agent.log*        |     ");
-
-    EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
-    appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
-        null, mockYarnClient, false, null, options, tags, null);
-
-    // Verify the include/exclude patterns
-    String expectedInclude = isOldApi ? "" : ".*";
-    String expectedExclude = "command*.json|agent.log*";
-    assertPatterns(expectedInclude, expectedExclude);
-
-    EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
-  }
-
-  @Test
-  public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern()
-      throws Exception {
-    Map<String, String> options = new HashMap<String, String>();
-    options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "");
-    options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "  ");
-
-    EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
-    appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
-        null, mockYarnClient, false, null, options, tags, null);
-
-    // Verify the include/exclude patterns
-    String expectedInclude = isOldApi ? "" : ".*";
-    String expectedExclude = "";
-    assertPatterns(expectedInclude, expectedExclude);
-
-    EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
-  }
-
-  private void assertPatterns(String expectedIncludePattern,
-      String expectedExcludePattern) throws Exception {
-    if (isOldApi) {
-      Assert.assertEquals(expectedIncludePattern,
-          appMasterLauncher.logAggregationContext.getIncludePattern());
-      Assert.assertEquals(expectedExcludePattern,
-          appMasterLauncher.logAggregationContext.getExcludePattern());
-    } else {
-      Assert.assertEquals(expectedIncludePattern,
-          (String) rolledLogsIncludeMethod
-              .invoke(appMasterLauncher.logAggregationContext));
-      Assert.assertEquals(expectedExcludePattern,
-          (String) rolledLogsExcludeMethod
-              .invoke(appMasterLauncher.logAggregationContext));
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
deleted file mode 100644
index a8f6b26..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.client.SliderYarnClientImpl;
-import org.apache.slider.common.SliderKeys;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestAppMasterLauncherWithAmReset {
-  SliderYarnClientImpl mockYarnClient;
-  YarnClientApplication yarnClientApp;
-  ApplicationSubmissionContext appSubmissionContext;
-  GetNewApplicationResponse newApp;
-  Set<String> tags = Collections.emptySet();
-  AppMasterLauncher appMasterLauncher = null;
-  boolean isOldApi = true;
-
-  @Before
-  public void initialize() throws Exception {
-    mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
-    yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
-    newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class);
-    EasyMock.expect(mockYarnClient.createApplication())
-        .andReturn(new YarnClientApplication(newApp,
-        Records.newRecord(ApplicationSubmissionContext.class)));
-  }
-
-  @Test
-  public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws
-      Exception {
-    Map<String, String> options = new HashMap<String, String>();
-    final String expectedInterval = Integer.toString (120000);
-    options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
-        expectedInterval);
-    EasyMock.replay(mockYarnClient, yarnClientApp);
-
-    appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
-        null, mockYarnClient, false, null, options, tags, null);
-
-    ApplicationSubmissionContext ctx = appMasterLauncher.application
-        .getApplicationSubmissionContext();
-    String retryIntervalWindow = Long.toString(ctx
-        .getAttemptFailuresValidityInterval());
-    Assert.assertEquals(expectedInterval, retryIntervalWindow);
-  }
-
-  @Test
-  public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue()
-      throws Exception {
-    Map<String, String> options = new HashMap<String, String>();
-    EasyMock.replay(mockYarnClient, yarnClientApp);
-
-    appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
-        null, mockYarnClient, false, null, options, tags, null);
-
-    ApplicationSubmissionContext ctx = appMasterLauncher.application
-        .getApplicationSubmissionContext();
-    long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval();
-    Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS,
-        retryIntervalWindow);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
deleted file mode 100644
index a1986cd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- *
- */
-public class TestServiceRecordAttributes extends Assert {
-
-  @Test
-  public void testAppConfigProvidedServiceRecordAttributes() throws Exception {
-    Map<String, String> options = new HashMap<>();
-    options.put("slider.some.arbitrary.option", "arbitrary value");
-    options.put("service.record.attribute.one_attribute", "one_attribute_value");
-    options.put("service.record.attribute.second_attribute", "second_attribute_value");
-    MapOperations serviceProps = new MapOperations(SliderKeys.COMPONENT_AM, options);
-    options = new HashMap<>();
-    options.put("some.component.attribute", "component_attribute_value");
-    options.put("service.record.attribute.component_attribute", "component_attribute_value");
-    MapOperations compProps = new MapOperations("TEST_COMP", options);
-
-    SliderAppMaster appMaster = new SliderAppMaster();
-
-    ServiceRecord appServiceRecord = new ServiceRecord();
-
-    appMaster.setProvidedServiceRecordAttributes(serviceProps, appServiceRecord);
-
-    assertNull("property should not be attribute",
-               appServiceRecord.get("slider.some.arbitrary.option"));
-    assertEquals("wrong value", "one_attribute_value",
-                 appServiceRecord.get("one_attribute"));
-    assertEquals("wrong value", "second_attribute_value",
-                 appServiceRecord.get("second_attribute"));
-
-    ServiceRecord compServiceRecord = new ServiceRecord();
-
-    appMaster.setProvidedServiceRecordAttributes(compProps, compServiceRecord);
-
-    assertNull("should not be attribute",
-               compServiceRecord.get("some.component.attribute"));
-    assertEquals("wrong value", "component_attribute_value",
-                 compServiceRecord.get("component_attribute"));
-
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message