hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ebad...@apache.org
Subject [hadoop] branch trunk updated: YARN-8805. Automatically convert the launch command to the exec form when using entrypoint support
Date Mon, 11 Mar 2019 17:55:31 GMT
This is an automated email from the ASF dual-hosted git repository.

ebadger pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new fa7a0b2  YARN-8805. Automatically convert the launch command to the exec form when
using entrypoint support
fa7a0b2 is described below

commit fa7a0b269a8613627819c41ab0e9c02a55e278b3
Author: Eric Badger <ebadger@verizonmedia.com>
AuthorDate: Mon Mar 11 12:43:14 2019 -0500

    YARN-8805. Automatically convert the launch command to the exec form when using entrypoint
support
---
 .../yarn/service/provider/ProviderUtils.java       | 17 ++++++++++++++
 .../provider/docker/DockerProviderService.java     |  6 +++++
 .../provider/TestAbstractProviderService.java      | 27 ++++++++++++++++++----
 .../yarn/service/provider/TestProviderUtils.java   |  9 ++++++++
 4 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 88883f7..ea1fb0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -47,9 +47,12 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.COMPONENT_ID;
@@ -149,6 +152,20 @@ public class ProviderUtils implements YarnServiceConstants {
     return content;
   }
 
+  public static String replaceSpacesWithDelimiter(String content,
+      String delimiter) {
+    List<String> parts = new ArrayList<String>();
+    Matcher m = Pattern.compile("([^\"]\\S*|\".+?\")\\s*").matcher(content);
+    while (m.find()) {
+      String part = m.group(1);
+      if(part.startsWith("\"") && part.endsWith("\"")) {
+        part = part.replaceAll("^\"|\"$", "");
+      }
+      parts.add(part);
+    }
+    return String.join(delimiter, parts);
+  }
+
   // configs will be substituted by corresponding env in tokenMap
   public static void substituteMapWithTokens(Map<String, String> configs,
       Map<String, String> tokenMap) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
index 6027a66..9b4138e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+
 import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
 import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
 import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
@@ -84,6 +85,11 @@ public class DockerProviderService extends AbstractProviderService
     if (useEntryPoint) {
       String launchCommand = compLaunchContext.getLaunchCommand();
       if (!StringUtils.isEmpty(launchCommand)) {
+        if(launchCommand.contains(" ")) {
+          // convert space delimiter command to exec format
+          launchCommand = ProviderUtils
+              .replaceSpacesWithDelimiter(launchCommand, ",");
+        }
         launcher.addCommand(launchCommand);
       }
     } else {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
index 81ccc7f..f0525aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestAbstractProviderService.java
@@ -82,7 +82,7 @@ public class TestAbstractProviderService {
     Component component = serviceContext.scheduler.getAllComponents().entrySet()
         .iterator().next().getValue();
     ContainerLaunchService.ComponentLaunchContext clc =
-        createEntryPointCLCFor(testService, component);
+        createEntryPointCLCFor(testService, component, "sleep,9000");
 
     ComponentInstance instance = component.getAllComponentInstances().iterator()
         .next();
@@ -96,12 +96,31 @@ public class TestAbstractProviderService {
   }
 
   @Test
+  public void testBuildContainerLaunchCommandWithSpace() throws Exception {
+    AbstractProviderService providerService = new DockerProviderService();
+    Component component = serviceContext.scheduler.getAllComponents().entrySet()
+        .iterator().next().getValue();
+    ContainerLaunchService.ComponentLaunchContext clc =
+        createEntryPointCLCFor(testService, component, "ls -l \" space\"");
+
+    ComponentInstance instance = component.getAllComponentInstances().iterator()
+        .next();
+    Container container = mock(Container.class);
+    providerService.buildContainerLaunchCommand(launcher, testService, instance,
+        rule.getFs(), serviceContext.scheduler.getConfig(), container, clc,
+        null);
+
+    Assert.assertEquals("commands don't match.",
+        Lists.newArrayList("ls,-l, space"), launcher.getCommands());
+  }
+
+  @Test
   public void testBuildContainerLaunchContext() throws Exception {
     AbstractProviderService providerService = new DockerProviderService();
     Component component = serviceContext.scheduler.getAllComponents().entrySet()
         .iterator().next().getValue();
     ContainerLaunchService.ComponentLaunchContext clc =
-        createEntryPointCLCFor(testService, component);
+        createEntryPointCLCFor(testService, component, "sleep,9000");
 
     ComponentInstance instance = component.getAllComponentInstances().iterator()
         .next();
@@ -118,8 +137,8 @@ public class TestAbstractProviderService {
   }
 
   private static ContainerLaunchService.ComponentLaunchContext
-  createEntryPointCLCFor(Service service, Component component) {
-    String launchCmd = "sleep,9000";
+      createEntryPointCLCFor(Service service, Component component,
+          String launchCmd) {
     Artifact artifact = new Artifact();
     artifact.setType(Artifact.TypeEnum.DOCKER);
     artifact.setId("example");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
index 0f7f375..84c3b6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/provider/TestProviderUtils.java
@@ -168,4 +168,13 @@ public class TestProviderUtils {
     Assert.assertEquals(resolved.getResolvedRsrcPaths().get("destFile1"),
         "destFile1");
   }
+
+  @Test
+  public void testReplaceSpacesWithDelimiter() {
+    String command = "ls  -l \" space\"";
+    String expected = "ls,-l, space";
+    String actual = ProviderUtils.replaceSpacesWithDelimiter(command, ",");
+    Assert.assertEquals("replaceSpaceWithDelimiter produces unexpected result.",
+        expected, actual);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message