From commits-return-12224-archive-asf-public=cust-asf.ponee.io@syncope.apache.org Mon Oct 8 16:00:28 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 89484180652 for ; Mon, 8 Oct 2018 16:00:27 +0200 (CEST) Received: (qmail 62362 invoked by uid 500); 8 Oct 2018 14:00:26 -0000 Mailing-List: contact commits-help@syncope.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@syncope.apache.org Delivered-To: mailing list commits@syncope.apache.org Received: (qmail 62353 invoked by uid 99); 8 Oct 2018 14:00:26 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 08 Oct 2018 14:00:26 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8A936E0016; Mon, 8 Oct 2018 14:00:26 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: fmartelli@apache.org To: commits@syncope.apache.org Message-Id: <3d951bf49e6e4f7b896f36f6bb361a7b@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: syncope git commit: [SYNCOPE-1379] provide the possibility to configure check connection timeouts (console side) + properties to customize resource connection checks thread pool Date: Mon, 8 Oct 2018 14:00:26 +0000 (UTC) Repository: syncope Updated Branches: refs/heads/master f0398c7d7 -> 1302734d3 [SYNCOPE-1379] provide the possibility to configure check connection timeouts (console side) + properties to customize resource connection checks thread pool Project: http://git-wip-us.apache.org/repos/asf/syncope/repo Commit: http://git-wip-us.apache.org/repos/asf/syncope/commit/1302734d Tree: http://git-wip-us.apache.org/repos/asf/syncope/tree/1302734d Diff: http://git-wip-us.apache.org/repos/asf/syncope/diff/1302734d Branch: refs/heads/master Commit: 1302734d34336558948205e0bfb17ecadaa24ede Parents: f0398c7 Author: fmartelli Authored: Mon Oct 8 15:55:42 2018 +0200 Committer: fmartelli Committed: Mon Oct 8 15:59:58 2018 +0200 ---------------------------------------------------------------------- .../console/SyncopeConsoleApplication.java | 23 ++ .../client/console/SyncopeConsoleSession.java | 6 +- .../topology/TopologyWebSocketBehavior.java | 211 +++++++++++++++---- .../src/main/resources/console.properties | 4 + .../src/main/resources/console.properties | 4 + .../src/test/resources/console.properties | 5 + 6 files changed, 211 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/syncope/blob/1302734d/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleApplication.java ---------------------------------------------------------------------- diff --git a/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleApplication.java b/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleApplication.java index cb866c0..2e15b19 100644 --- a/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleApplication.java +++ b/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleApplication.java @@ -104,6 +104,12 @@ public class SyncopeConsoleApplication extends AuthenticatedWebApplication { private Integer maxWaitTime; + private Integer corePoolSize; + + private Integer maxPoolSize; + + private Integer queueCapacity; + private List domains; private Map> pageClasses; @@ -159,6 +165,11 @@ public class SyncopeConsoleApplication extends AuthenticatedWebApplication { maxWaitTime = Integer.valueOf(props.getProperty("maxWaitTimeOnApplyChanges", "30")); + // Resource connections check thread pool size + corePoolSize = Integer.valueOf(props.getProperty("topology.corePoolSize", "5")); + maxPoolSize = Integer.valueOf(props.getProperty("topology.maxPoolSize", "10")); + queueCapacity = Integer.valueOf(props.getProperty("topology.queueCapacity", "50")); + String csrf = props.getProperty("csrf"); // process page properties @@ -276,6 +287,18 @@ public class SyncopeConsoleApplication extends AuthenticatedWebApplication { return maxWaitTime; } + public Integer getCorePoolSize() { + return corePoolSize; + } + + public Integer getMaxPoolSize() { + return maxPoolSize; + } + + public Integer getQueueCapacity() { + return queueCapacity; + } + public SyncopeClientFactoryBean newClientFactory() { return new SyncopeClientFactoryBean(). setAddress(scheme + "://" + host + ":" + port + StringUtils.prependIfMissing(rootPath, "/")). http://git-wip-us.apache.org/repos/asf/syncope/blob/1302734d/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleSession.java ---------------------------------------------------------------------- diff --git a/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleSession.java b/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleSession.java index 2f952ee..d793da2 100644 --- a/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleSession.java +++ b/client/console/src/main/java/org/apache/syncope/client/console/SyncopeConsoleSession.java @@ -104,9 +104,9 @@ public class SyncopeConsoleSession extends AuthenticatedWebSession { executor = new ThreadPoolTaskExecutor(); executor.setWaitForTasksToCompleteOnShutdown(false); - executor.setCorePoolSize(5); - executor.setMaxPoolSize(10); - executor.setQueueCapacity(50); + executor.setCorePoolSize(SyncopeConsoleApplication.get().getCorePoolSize()); + executor.setMaxPoolSize(SyncopeConsoleApplication.get().getMaxPoolSize()); + executor.setQueueCapacity(SyncopeConsoleApplication.get().getQueueCapacity()); executor.initialize(); } http://git-wip-us.apache.org/repos/asf/syncope/blob/1302734d/client/console/src/main/java/org/apache/syncope/client/console/topology/TopologyWebSocketBehavior.java ---------------------------------------------------------------------- diff --git a/client/console/src/main/java/org/apache/syncope/client/console/topology/TopologyWebSocketBehavior.java b/client/console/src/main/java/org/apache/syncope/client/console/topology/TopologyWebSocketBehavior.java index 1b1a448..320e52c 100644 --- a/client/console/src/main/java/org/apache/syncope/client/console/topology/TopologyWebSocketBehavior.java +++ b/client/console/src/main/java/org/apache/syncope/client/console/topology/TopologyWebSocketBehavior.java @@ -30,9 +30,15 @@ import java.util.Set; import javax.servlet.SessionTrackingMode; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.syncope.client.console.SyncopeConsoleSession; +import org.apache.syncope.client.console.rest.ConfRestClient; import org.apache.syncope.client.console.rest.ConnectorRestClient; import org.apache.syncope.client.console.rest.ResourceRestClient; +import org.apache.syncope.common.lib.SyncopeClientException; +import org.apache.syncope.common.lib.to.AttrTO; import org.apache.syncope.common.lib.to.ConnInstanceTO; import org.apache.syncope.common.lib.to.ResourceTO; import org.apache.wicket.Application; @@ -46,6 +52,7 @@ import org.apache.wicket.protocol.ws.api.message.TextMessage; import org.apache.wicket.util.cookies.CookieUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; public class TopologyWebSocketBehavior extends WebSocketBehavior { @@ -55,13 +62,19 @@ public class TopologyWebSocketBehavior extends WebSocketBehavior { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final Map resources = - Collections.synchronizedMap(new HashMap<>()); + private final Map resources = Collections.synchronizedMap(new HashMap<>()); + + private static final String CONNECTOR_TEST_TIMEOUT_PARAMETER = "connector.test.timeout"; + + private Integer connectorTestTimeout = null; + + private static final String RESOURCE_TEST_TIMEOUT_PARAMETER = "resource.test.timeout"; + + private Integer resourceTestTimeout = null; private final Set runningResCheck = Collections.synchronizedSet(new HashSet<>()); - private final Map connectors = - Collections.synchronizedMap(new HashMap<>()); + private final Map connectors = Collections.synchronizedMap(new HashMap<>()); private final Set runningConnCheck = Collections.synchronizedSet(new HashSet<>()); @@ -69,12 +82,34 @@ public class TopologyWebSocketBehavior extends WebSocketBehavior { private final ResourceRestClient resourceRestClient = new ResourceRestClient(); + public TopologyWebSocketBehavior() { + // Handling with timeout as per SYNCOPE-1379 + try { + // Loop just to avoid NotFound exception raising on the Core side + for (AttrTO param : new ConfRestClient().list()) { + if (!CollectionUtils.isEmpty(param.getValues())) { + try { + if (CONNECTOR_TEST_TIMEOUT_PARAMETER.equalsIgnoreCase(param.getSchema())) { + connectorTestTimeout = Integer.parseInt(param.getValues().get(0)); + } else if (RESOURCE_TEST_TIMEOUT_PARAMETER.equalsIgnoreCase(param.getSchema())) { + resourceTestTimeout = Integer.parseInt(param.getValues().get(0)); + } + } catch (NumberFormatException e) { + LOG.warn("Invalid timeout {}", param); + } + } + } + } catch (SyncopeClientException e) { + // ignore exception + } + } + @Override protected CharSequence getSessionId(final Component component) { String sessionId = ""; WebApplication application = (WebApplication) component.getApplication(); - Set effectiveSessionTrackingModes = - application.getServletContext().getEffectiveSessionTrackingModes(); + Set effectiveSessionTrackingModes = application.getServletContext(). + getEffectiveSessionTrackingModes(); Object containerRequest = component.getRequest().getContainerRequest(); if (effectiveSessionTrackingModes.size() == 1 && SessionTrackingMode.URL.equals(effectiveSessionTrackingModes.iterator().next())) { @@ -120,7 +155,11 @@ public class TopologyWebSocketBehavior extends WebSocketBehavior { runningConnCheck.add(ckey); } - SyncopeConsoleSession.get().execute(new ConnCheck(ckey)); + try { + SyncopeConsoleSession.get().execute(new ConnCheck(ckey)); + } catch (Exception e) { + LOG.error("Unexpected error", e); + } break; case CHECK_RESOURCE: @@ -139,7 +178,11 @@ public class TopologyWebSocketBehavior extends WebSocketBehavior { runningResCheck.add(rkey); } - SyncopeConsoleSession.get().execute(new ResCheck(rkey)); + try { + SyncopeConsoleSession.get().execute(new ResCheck(rkey)); + } catch (Exception e) { + LOG.error("Unexpected error", e); + } break; case ADD_ENDPOINT: @@ -163,6 +206,37 @@ public class TopologyWebSocketBehavior extends WebSocketBehavior { return this.resources.keySet().containsAll(resources); } + private void timeoutHandlingConnectionChecker( + final Checker checker, + final Integer timeout, + final Map responses, + final Set running) { + String res = null; + try { + if (timeout == null) { + LOG.debug("No timeouts for resource connection checking ... "); + res = SyncopeConsoleSession.get().execute(checker).get(); + } else if (timeout > 0) { + LOG.debug("Timeouts provided for resource connection checking ... "); + res = SyncopeConsoleSession.get().execute(checker).get(timeout, TimeUnit.SECONDS); + } + } catch (InterruptedException | TimeoutException e) { + LOG.warn("Connection with {} timed out", checker.getKey()); + res = String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", + TopologyNode.Status.UNREACHABLE, checker.getKey()); + } catch (Exception e) { + LOG.error("Unexpected exception conneting to {}", checker.getKey(), e); + res = String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", + TopologyNode.Status.FAILURE, checker.getKey()); + } + + if (res != null) { + responses.put(checker.getKey(), res); + } + + running.remove(checker.getKey()); + } + class ConnCheck implements Runnable { private final String key; @@ -179,23 +253,15 @@ public class TopologyWebSocketBehavior extends WebSocketBehavior { @Override public void run() { - try { - ThreadContext.setApplication(application); - ThreadContext.setSession(session); - - String res; - try { - final ConnInstanceTO connector = connectorRestClient.read(key); - res = String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", - connectorRestClient.check(connector).getLeft() - ? TopologyNode.Status.REACHABLE : TopologyNode.Status.UNREACHABLE, key); - } catch (Exception e) { - LOG.warn("Error checking connection for {}", key, e); - res = String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", TopologyNode.Status.FAILURE, key); - } + ThreadContext.setApplication(application); + ThreadContext.setSession(session); - connectors.put(key, res); - runningConnCheck.remove(key); + try { + timeoutHandlingConnectionChecker( + new ConnectorChecker(key, this.application), + connectorTestTimeout, + connectors, + runningConnCheck); } finally { ThreadContext.detach(); } @@ -218,23 +284,90 @@ public class TopologyWebSocketBehavior extends WebSocketBehavior { @Override public void run() { + ThreadContext.setApplication(application); + ThreadContext.setSession(session); + try { - ThreadContext.setApplication(application); - ThreadContext.setSession(session); - - String res; - try { - final ResourceTO resource = resourceRestClient.read(key); - res = String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", - resourceRestClient.check(resource).getLeft() - ? TopologyNode.Status.REACHABLE : TopologyNode.Status.UNREACHABLE, key); - } catch (Exception e) { - LOG.warn("Error checking connection for {}", key, e); - res = String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", TopologyNode.Status.FAILURE, key); - } + timeoutHandlingConnectionChecker( + new ResourceChecker(key, this.application), + resourceTestTimeout, + resources, + runningResCheck); + } finally { + ThreadContext.detach(); + } + } + } - resources.put(key, res); - runningResCheck.remove(key); + abstract class Checker implements Callable { + + protected final String key; + + protected final Application application; + + protected final Session session; + + Checker(final String key, final Application application) { + this.key = key; + this.application = application; + this.session = Session.exists() ? Session.get() : null; + } + + public String getKey() { + return key; + } + + @Override + public abstract String call() throws Exception; + } + + class ConnectorChecker extends Checker { + + ConnectorChecker(final String key, final Application application) { + super(key, application); + } + + @Override + public String call() throws Exception { + ThreadContext.setApplication(application); + ThreadContext.setSession(session); + + try { + final ConnInstanceTO connector = connectorRestClient.read(key); + return String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", + connectorRestClient.check(connector).getLeft() + ? TopologyNode.Status.REACHABLE : TopologyNode.Status.UNREACHABLE, key); + } catch (Exception e) { + LOG.warn("Error checking connection for {}", key, e); + return String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", + TopologyNode.Status.FAILURE, key); + } finally { + ThreadContext.detach(); + } + } + } + + class ResourceChecker extends Checker { + + ResourceChecker(final String key, final Application application) { + super(key, application); + } + + @Override + public String call() throws Exception { + ThreadContext.setApplication(application); + ThreadContext.setSession(session); + + try { + final ResourceTO resource = resourceRestClient.read(key); + return String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", + resourceRestClient.check(resource).getLeft() + ? TopologyNode.Status.REACHABLE : TopologyNode.Status.UNREACHABLE, key); + } catch (Exception e) { + LOG.warn("Error checking connection for {}", key, e); + return String.format("{ \"status\": \"%s\", \"target\": \"%s\"}", + TopologyNode.Status.FAILURE, + key); } finally { ThreadContext.detach(); } http://git-wip-us.apache.org/repos/asf/syncope/blob/1302734d/client/console/src/main/resources/console.properties ---------------------------------------------------------------------- diff --git a/client/console/src/main/resources/console.properties b/client/console/src/main/resources/console.properties index f8a6467..756e71e 100644 --- a/client/console/src/main/resources/console.properties +++ b/client/console/src/main/resources/console.properties @@ -47,3 +47,7 @@ page.types=org.apache.syncope.client.console.pages.Types page.policies=org.apache.syncope.client.console.pages.Policies page.notifications=org.apache.syncope.client.console.pages.Notifications page.parameters=org.apache.syncope.client.console.pages.Parameters + +topology.corePoolSize=10 +topology.maxPoolSize=20 +topology.queueCapacity=50 http://git-wip-us.apache.org/repos/asf/syncope/blob/1302734d/fit/console-reference/src/main/resources/console.properties ---------------------------------------------------------------------- diff --git a/fit/console-reference/src/main/resources/console.properties b/fit/console-reference/src/main/resources/console.properties index 484a331..310ceff 100644 --- a/fit/console-reference/src/main/resources/console.properties +++ b/fit/console-reference/src/main/resources/console.properties @@ -47,3 +47,7 @@ page.types=org.apache.syncope.client.console.pages.Types page.policies=org.apache.syncope.client.console.pages.Policies page.notifications=org.apache.syncope.client.console.pages.Notifications page.parameters=org.apache.syncope.client.console.pages.Parameters + +topology.corePoolSize=50 +topology.maxPoolSize=100 +topology.queueCapacity=10 http://git-wip-us.apache.org/repos/asf/syncope/blob/1302734d/fit/core-reference/src/test/resources/console.properties ---------------------------------------------------------------------- diff --git a/fit/core-reference/src/test/resources/console.properties b/fit/core-reference/src/test/resources/console.properties index 6e47934..eb45235 100644 --- a/fit/core-reference/src/test/resources/console.properties +++ b/fit/core-reference/src/test/resources/console.properties @@ -47,3 +47,8 @@ page.types=org.apache.syncope.client.console.pages.Types page.policies=org.apache.syncope.client.console.pages.Policies page.notifications=org.apache.syncope.client.console.pages.Notifications page.parameters=org.apache.syncope.client.console.pages.Parameters + +topology.corePoolSize=50 +topology.maxPoolSize=100 +topology.queueCapacity=10 +