flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (FLINK-8317) Enable Triggering of Savepoints via RestfulGateway
Date Tue, 09 Jan 2018 15:11:00 GMT

    [ https://issues.apache.org/jira/browse/FLINK-8317?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16318563#comment-16318563
] 

ASF GitHub Bot commented on FLINK-8317:
---------------------------------------

Github user GJL commented on a diff in the pull request:

    https://github.com/apache/flink/pull/5223#discussion_r160431822
  
    --- Diff: flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java
---
    @@ -0,0 +1,276 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.flink.runtime.rest.handler.job.savepoints;
    +
    +import org.apache.flink.api.common.JobID;
    +import org.apache.flink.api.common.time.Time;
    +import org.apache.flink.core.fs.Path;
    +import org.apache.flink.runtime.checkpoint.CheckpointProperties;
    +import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
    +import org.apache.flink.runtime.concurrent.FutureUtils;
    +import org.apache.flink.runtime.rest.handler.HandlerRequest;
    +import org.apache.flink.runtime.rest.handler.HandlerRequestException;
    +import org.apache.flink.runtime.rest.handler.RestHandlerException;
    +import org.apache.flink.runtime.rest.messages.EmptyRequestBody;
    +import org.apache.flink.runtime.rest.messages.JobIDPathParameter;
    +import org.apache.flink.runtime.rest.messages.SavepointTriggerIdPathParameter;
    +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointResponseBody;
    +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointStatusMessageParameters;
    +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerId;
    +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerMessageParameters;
    +import org.apache.flink.runtime.rest.messages.job.savepoints.SavepointTriggerRequestBody;
    +import org.apache.flink.runtime.rest.messages.queue.QueueStatus;
    +import org.apache.flink.runtime.state.filesystem.FileStateHandle;
    +import org.apache.flink.runtime.webmonitor.RestfulGateway;
    +import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
    +
    +import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseStatus;
    +
    +import org.junit.Before;
    +import org.junit.Test;
    +import org.mockito.ArgumentCaptor;
    +import org.mockito.Mock;
    +import org.mockito.MockitoAnnotations;
    +
    +import java.util.Collections;
    +import java.util.HashMap;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.concurrent.CompletableFuture;
    +import java.util.concurrent.ExecutionException;
    +
    +import static org.hamcrest.Matchers.containsString;
    +import static org.hamcrest.Matchers.equalTo;
    +import static org.hamcrest.Matchers.hasItem;
    +import static org.hamcrest.Matchers.instanceOf;
    +import static org.hamcrest.Matchers.notNullValue;
    +import static org.junit.Assert.assertThat;
    +import static org.junit.Assert.fail;
    +import static org.mockito.Matchers.any;
    +import static org.mockito.Matchers.anyString;
    +import static org.mockito.Mockito.when;
    +
    +/**
    + * Test for {@link SavepointHandlers}.
    + */
    +public class SavepointHandlersTest {
    +
    +	private static final CompletableFuture<String> LOCAL_REST_ADDRESS =
    +		CompletableFuture.completedFuture("localhost:12345");
    +
    +	private static final Time TIMEOUT = Time.seconds(10);
    +
    +	private static final JobID JOB_ID = new JobID();
    +
    +	private static final String COMPLETED_CHECKPOINT_EXTERNAL_POINTER = "/tmp/savepoint-0d2fb9-8d5e0106041a";
    +
    +	private static final String DEFAULT_REQUESTED_SAVEPOINT_TARGET_DIRECTORY = "/tmp";
    +
    +	@Mock
    +	private RestfulGateway mockRestfulGateway;
    +
    +	private SavepointHandlers savepointHandlers;
    +
    +	private SavepointHandlers.SavepointTriggerHandler savepointTriggerHandler;
    +
    +	private SavepointHandlers.SavepointStatusHandler savepointStatusHandler;
    +
    +	private CompletedCheckpoint completedCheckpoint;
    +
    +	@Before
    +	public void setUp() throws Exception {
    +		MockitoAnnotations.initMocks(this);
    +
    +		final GatewayRetriever<RestfulGateway> leaderRetriever = new GatewayRetriever<RestfulGateway>()
{
    +			@Override
    +			public CompletableFuture<RestfulGateway> getFuture() {
    +				return CompletableFuture.completedFuture(mockRestfulGateway);
    +			}
    +		};
    +
    +		savepointHandlers = new SavepointHandlers(null);
    +		savepointTriggerHandler = savepointHandlers.new SavepointTriggerHandler(
    +			LOCAL_REST_ADDRESS,
    +			leaderRetriever,
    +			TIMEOUT,
    +			Collections.emptyMap());
    +
    +		savepointStatusHandler = savepointHandlers.new SavepointStatusHandler(
    +			LOCAL_REST_ADDRESS,
    +			leaderRetriever,
    +			TIMEOUT,
    +			Collections.emptyMap());
    +
    +		completedCheckpoint = new CompletedCheckpoint(
    +			new JobID(),
    +			0,
    +			0,
    +			0,
    +			Collections.emptyMap(),
    +			null,
    +			CheckpointProperties.forStandardSavepoint(),
    +			new FileStateHandle(new Path("/tmp"), 0),
    +			COMPLETED_CHECKPOINT_EXTERNAL_POINTER
    +		);
    +	}
    +
    +	@Test
    +	public void testSavepointCompletedSuccessfully() throws Exception {
    +		final CompletableFuture<CompletedCheckpoint> checkpointCompletableFuture =
    +			new CompletableFuture<>();
    +		when(mockRestfulGateway.triggerSavepoint(any(JobID.class), anyString(), any(Time.class)))
    +			.thenReturn(checkpointCompletableFuture);
    +
    +		final SavepointTriggerId savepointTriggerId = savepointTriggerHandler.handleRequest(
    +			triggerSavepointRequest(),
    +			mockRestfulGateway).get().getSavepointTriggerId();
    +
    +		SavepointResponseBody savepointResponseBody;
    +		savepointResponseBody = savepointStatusHandler.handleRequest(
    +			savepointStatusRequest(savepointTriggerId),
    +			mockRestfulGateway).get();
    +
    +		assertThat(
    +			savepointResponseBody.getStatus().getStatusId(),
    +			equalTo(QueueStatus.StatusId.IN_PROGRESS));
    +
    +		checkpointCompletableFuture.complete(completedCheckpoint);
    +		savepointResponseBody = savepointStatusHandler.handleRequest(
    +			savepointStatusRequest(savepointTriggerId),
    +			mockRestfulGateway).get();
    +
    +		assertThat(
    +			savepointResponseBody.getStatus().getStatusId(),
    +			equalTo(QueueStatus.StatusId.COMPLETED));
    +		assertThat(savepointResponseBody.getSavepoint(), notNullValue());
    +		assertThat(
    +			savepointResponseBody.getSavepoint().getLocation(),
    +			equalTo(COMPLETED_CHECKPOINT_EXTERNAL_POINTER));
    +	}
    +
    +	@Test
    +	public void testTriggerSavepointWithDefaultDirectory() throws Exception {
    +		final ArgumentCaptor<String> targetDirectoryCaptor = ArgumentCaptor.forClass(String.class);
    +		when(mockRestfulGateway.triggerSavepoint(any(JobID.class), targetDirectoryCaptor.capture(),
any(Time.class)))
    +			.thenReturn(CompletableFuture.completedFuture(completedCheckpoint));
    +		final String defaultSavepointDir = "/other/dir";
    +		savepointHandlers.setDefaultSavepointDir(defaultSavepointDir);
    +
    +		savepointTriggerHandler.handleRequest(
    +			triggerSavepointRequestWithDefaultDirectory(),
    +			mockRestfulGateway).get();
    +
    +		final List<String> targetDirectories = targetDirectoryCaptor.getAllValues();
    +		assertThat(targetDirectories, hasItem(equalTo(defaultSavepointDir)));
    +	}
    +
    +	@Test
    +	public void testGetSavepointStatusWithUnknownSavepointTriggerId() throws Exception {
    +		final SavepointTriggerId unknownSavepointTriggerId = new SavepointTriggerId();
    +
    +		try {
    +			savepointStatusHandler.handleRequest(
    +				savepointStatusRequest(unknownSavepointTriggerId),
    +				mockRestfulGateway).get();
    +			fail("Expected exception not thrown.");
    +		} catch (ExecutionException e) {
    +			final Throwable cause = e.getCause();
    +			assertThat(cause, instanceOf(RestHandlerException.class));
    +			final RestHandlerException restHandlerException = (RestHandlerException) cause;
    +			assertThat(restHandlerException.getMessage(), containsString("Savepoint not found"));
    +			assertThat(restHandlerException.getHttpResponseStatus(), equalTo(HttpResponseStatus.NOT_FOUND));
    +		}
    +	}
    +
    +	@Test
    +	public void testTriggerSavepointNoDirectory() throws Exception {
    +		when(mockRestfulGateway.triggerSavepoint(any(JobID.class), anyString(), any(Time.class)))
    +			.thenReturn(CompletableFuture.completedFuture(completedCheckpoint));
    +
    +		try {
    +			savepointTriggerHandler.handleRequest(
    +				triggerSavepointRequestWithDefaultDirectory(),
    +				mockRestfulGateway).get();
    +		} catch (ExecutionException e) {
    --- End diff --
    
    missing `fail()`


> Enable Triggering of Savepoints via RestfulGateway
> --------------------------------------------------
>
>                 Key: FLINK-8317
>                 URL: https://issues.apache.org/jira/browse/FLINK-8317
>             Project: Flink
>          Issue Type: New Feature
>          Components: Distributed Coordination, REST
>    Affects Versions: 1.5.0
>            Reporter: Gary Yao
>            Assignee: Gary Yao
>              Labels: flip-6
>             Fix For: 1.5.0
>
>
> Enable triggering of savepoints in FLIP-6 mode via RestfulGateway:
> * Add method to {{CompletableFuture<CompletedCheckpoint> triggerSavepoint(long
timestamp, String targetDirectory)}} to interface
> * Implement method in {{Dispatcher}} and {{JobMaster}}



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Mime
View raw message