flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (FLINK-10295) Tokenisation of Program Args resulting in unexpected results
Date Mon, 08 Oct 2018 11:49:01 GMT

    [ https://issues.apache.org/jira/browse/FLINK-10295?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16641707#comment-16641707
] 

ASF GitHub Bot commented on FLINK-10295:
----------------------------------------

zentol commented on a change in pull request #6754: [FLINK-10295] Add support of passing jar
arguments as list of separate strings in REST API
URL: https://github.com/apache/flink/pull/6754#discussion_r223328469
 
 

 ##########
 File path: flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarHandlerParameterTest.java
 ##########
 @@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.webmonitor.handlers;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.time.Time;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.messages.Acknowledge;
+import org.apache.flink.runtime.rest.handler.HandlerRequest;
+import org.apache.flink.runtime.rest.handler.HandlerRequestException;
+import org.apache.flink.runtime.rest.handler.RestHandlerException;
+import org.apache.flink.runtime.rest.messages.MessageParameter;
+import org.apache.flink.runtime.rest.messages.MessageQueryParameter;
+import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.runtime.util.BlobServerResource;
+import org.apache.flink.runtime.webmonitor.TestingDispatcherGateway;
+import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
+import org.apache.flink.runtime.webmonitor.testutils.ParameterProgram;
+import org.apache.flink.util.TestLogger;
+
+import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.fail;
+
+/** Base test class for jar request handlers. */
+public abstract class JarHandlerParameterTest
+	<REQB extends JarRequestBody, M extends JarMessageParameters> extends TestLogger {
+	enum ProgramArgsParType {
+		String,
+		List,
+		Both
+	}
+
+	static final String[] PROG_ARGS = new String[] {"--host", "localhost", "--port", "1234"};
+	static final int PARALLELISM = 4;
+
+	@ClassRule
+	public static final TemporaryFolder TMP = new TemporaryFolder();
+
+	@ClassRule
+	public static final BlobServerResource BLOB_SERVER_RESOURCE = new BlobServerResource();
+
+	static final AtomicReference<JobGraph> LAST_SUBMITTED_JOB_GRAPH_REFERENCE = new AtomicReference<>();
+
+	static TestingDispatcherGateway restfulGateway;
+	static Path jarDir;
+	static GatewayRetriever<TestingDispatcherGateway> gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway);
+	static CompletableFuture<String> localAddressFuture = CompletableFuture.completedFuture("shazam://localhost:12345");
+	static Time timeout = Time.seconds(10);
+	static Map<String, String> responseHeaders = Collections.emptyMap();
+	static Executor executor = TestingUtils.defaultExecutor();
+
+	private static Path jarWithManifest;
+	private static Path jarWithoutManifest;
+
+	static void init() throws Exception {
+		jarDir = TMP.newFolder().toPath();
+
+		// properties are set property by surefire plugin
+		final String parameterProgramJarName = System.getProperty("parameterJarName") + ".jar";
+		final String parameterProgramWithoutManifestJarName = System.getProperty("parameterJarWithoutManifestName")
+ ".jar";
+		final Path jarLocation = Paths.get(System.getProperty("targetDir"));
+
+		jarWithManifest = Files.copy(
+			jarLocation.resolve(parameterProgramJarName),
+			jarDir.resolve("program-with-manifest.jar"));
+		jarWithoutManifest = Files.copy(
+			jarLocation.resolve(parameterProgramWithoutManifestJarName),
+			jarDir.resolve("program-without-manifest.jar"));
+
+		restfulGateway = new TestingDispatcherGateway.Builder()
+			.setBlobServerPort(BLOB_SERVER_RESOURCE.getBlobServerPort())
+			.setSubmitFunction(jobGraph -> {
+				LAST_SUBMITTED_JOB_GRAPH_REFERENCE.set(jobGraph);
+				return CompletableFuture.completedFuture(Acknowledge.get());
+			})
+			.build();
+
+		gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway);
+		localAddressFuture = CompletableFuture.completedFuture("shazam://localhost:12345");
+		timeout = Time.seconds(10);
+		responseHeaders = Collections.emptyMap();
+		executor = TestingUtils.defaultExecutor();
+	}
+
+	@Before
+	public void reset() {
+		ParameterProgram.actualArguments = null;
+	}
+
+	@Test
+	public void testDefaultParameters() throws Exception {
+		// baseline, ensure that reasonable defaults are chosen
+		handleRequest(createRequest(
+			getDefaultJarRequestBody(),
+			getUnresolvedJarMessageParameters(),
+			getUnresolvedJarMessageParameters(),
+			jarWithManifest));
+		validateDefaultGraph();
+	}
+
+	@Test
+	public void testConfigurationViaQueryParametersWithProgArgsAsString() throws Exception {
+		testConfigurationViaQueryParameters(ProgramArgsParType.String);
+	}
+
+	@Test
+	public void testConfigurationViaQueryParametersWithProgArgsAsList() throws Exception {
+		testConfigurationViaQueryParameters(ProgramArgsParType.List);
+	}
+
+	@Test
+	public void testConfigurationViaQueryParametersFailWithProgArgsAsStringAndList() throws
Exception {
+		try {
+			testConfigurationViaQueryParameters(ProgramArgsParType.Both);
+			fail("RestHandlerException is excepted");
+		} catch (RestHandlerException e) {
+			assertEquals(HttpResponseStatus.BAD_REQUEST, e.getHttpResponseStatus());
+		}
+	}
+
+	private void testConfigurationViaQueryParameters(ProgramArgsParType programArgsParType)
throws Exception {
+		// configure submission via query parameters
+		handleRequest(createRequest(
+			getDefaultJarRequestBody(),
+			getJarMessageParameters(programArgsParType),
+			getUnresolvedJarMessageParameters(),
+			jarWithoutManifest));
+		validateGraph();
+	}
+
+	@Test
+	public void testConfigurationViaJsonRequestWithProgArgsAsString() throws Exception {
+		testConfigurationViaJsonRequest(ProgramArgsParType.String);
+	}
+
+	@Test
+	public void testConfigurationViaJsonRequestWithProgArgsAsList() throws Exception {
+		testConfigurationViaJsonRequest(ProgramArgsParType.List);
+	}
+
+	@Test
+	public void testConfigurationViaJsonRequestFailWithProgArgsAsStringAndList() throws Exception
{
+		try {
+			testConfigurationViaJsonRequest(ProgramArgsParType.Both);
+			fail("RestHandlerException is excepted");
+		} catch (RestHandlerException e) {
+			assertEquals(HttpResponseStatus.BAD_REQUEST, e.getHttpResponseStatus());
+		}
+	}
+
+	private void testConfigurationViaJsonRequest(ProgramArgsParType programArgsParType) throws
Exception {
+		handleRequest(createRequest(
+			getJarRequestBody(programArgsParType),
+			getUnresolvedJarMessageParameters(),
+			getUnresolvedJarMessageParameters(),
+			jarWithoutManifest
+		));
+		validateGraph();
+	}
+
+	@Test
+	public void testParameterPrioritizationWithProgArgsAsString() throws Exception {
+		testParameterPrioritization(ProgramArgsParType.String);
+	}
+
+	@Test
+	public void testParameterPrioritizationWithProgArgsAsList() throws Exception {
+		testParameterPrioritization(ProgramArgsParType.List);
+	}
+
+	@Test
+	public void testFailIfProgArgsAreAsStringAndAsList() throws Exception {
+		try {
+			testParameterPrioritization(ProgramArgsParType.Both);
+			fail("RestHandlerException is excepted");
+		} catch (RestHandlerException e) {
+			assertEquals(HttpResponseStatus.BAD_REQUEST, e.getHttpResponseStatus());
+		}
+	}
+
+	private void testParameterPrioritization(ProgramArgsParType programArgsParType) throws Exception
{
+		// configure submission via query parameters and JSON request, JSON should be prioritized
+		handleRequest(createRequest(
+			getJarRequestBody(programArgsParType),
+			getWrongJarMessageParameters(programArgsParType),
+			getUnresolvedJarMessageParameters(),
+			jarWithoutManifest));
+		validateGraph();
+	}
+
+	static String getProgramArgsString(ProgramArgsParType programArgsParType) {
+		return programArgsParType == ProgramArgsParType.String || programArgsParType == ProgramArgsParType.Both
+			? String.join(" ", PROG_ARGS) : null;
+	}
+
+	static List<String> getProgramArgsList(ProgramArgsParType programArgsParType) {
+		return programArgsParType == ProgramArgsParType.List || programArgsParType == ProgramArgsParType.Both
+			? Arrays.asList(PROG_ARGS) : null;
+	}
+
+	private static <
+		REQB extends JarRequestBody,
+		M extends JarMessageParameters>
+	HandlerRequest<REQB, M> createRequest(
 
 Review comment:
   the indentation doesn't look correct, it should not be in the same column as `private static
<`

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Tokenisation of Program Args resulting in unexpected results
> ------------------------------------------------------------
>
>                 Key: FLINK-10295
>                 URL: https://issues.apache.org/jira/browse/FLINK-10295
>             Project: Flink
>          Issue Type: Bug
>          Components: REST, Webfrontend
>    Affects Versions: 1.5.0, 1.6.0, 1.7.0
>            Reporter: Gaurav Singhania
>            Assignee: Andrey Zagrebin
>            Priority: Blocker
>              Labels: pull-request-available
>             Fix For: 1.7.0
>
>         Attachments: sample_request.txt
>
>
> We were upgrading from Flink 1.4 to 1.6. At present we have a jar which takes all the
details to run the job as program args against a jarid, including sql query and kafka details.
In version 1.5 the program args are tokenised as a result single quote (') and double quote(")
are stripped from the arguments. This results in malformed args.
> Attached a sample request for reference.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message