brooklyn-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (BROOKLYN-6) Persistence to object store fails with 401 unauthorized after some time
Date Thu, 26 Jun 2014 09:57:25 GMT

    [ https://issues.apache.org/jira/browse/BROOKLYN-6?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14044528#comment-14044528
] 

ASF GitHub Bot commented on BROOKLYN-6:
---------------------------------------

Github user aledsage commented on a diff in the pull request:

    https://github.com/apache/incubator-brooklyn/pull/19#discussion_r14232802
  
    --- Diff: locations/jclouds/src/main/java/brooklyn/location/jclouds/config/AlwaysRetryOnRenew.java
---
    @@ -0,0 +1,121 @@
    +package brooklyn.location.jclouds.config;
    +
    +import static org.jclouds.http.HttpUtils.closeClientButKeepContentStream;
    +import static org.jclouds.http.HttpUtils.releasePayload;
    +
    +import java.lang.reflect.Method;
    +
    +import javax.annotation.Resource;
    +
    +import org.aopalliance.intercept.MethodInterceptor;
    +import org.aopalliance.intercept.MethodInvocation;
    +import org.jclouds.domain.Credentials;
    +import org.jclouds.http.HttpCommand;
    +import org.jclouds.http.HttpResponse;
    +import org.jclouds.http.HttpRetryHandler;
    +import org.jclouds.logging.Logger;
    +import org.jclouds.openstack.domain.AuthenticationResponse;
    +import org.jclouds.openstack.handlers.RetryOnRenew;
    +import org.jclouds.openstack.reference.AuthHeaders;
    +
    +import com.google.common.annotations.Beta;
    +import com.google.common.cache.LoadingCache;
    +import com.google.common.collect.Multimap;
    +import com.google.inject.AbstractModule;
    +import com.google.inject.Inject;
    +import com.google.inject.Injector;
    +import com.google.inject.Singleton;
    +import com.google.inject.matcher.AbstractMatcher;
    +import com.google.inject.matcher.Matcher;
    +import com.google.inject.matcher.Matchers;
    +
    +/** Fix for RetryOnRenew so that it always retries on 401 when using a token.
    + *  The "lease renew" text is not necessarily returned from swift servers.
    + *  <p>
    + *  See https://issues.apache.org/jira/browse/BROOKLYN-6 .
    + *  When https://issues.apache.org/jira/browse/JCLOUDS-615 is fixed in the jclouds we
use,
    + *  we can remove this. 
    + *  <p>
    + *  (Marked Beta as this will likely be removed.)
    + *  
    + *  @since 1.7.0 */
    +@Beta
    +@Singleton
    +public class AlwaysRetryOnRenew implements HttpRetryHandler {
    +   @Resource
    +   protected Logger logger = Logger.NULL;
    +
    +   private final LoadingCache<Credentials, AuthenticationResponse> authenticationResponseCache;
    +
    +   @Inject
    +   protected AlwaysRetryOnRenew(LoadingCache<Credentials, AuthenticationResponse>
authenticationResponseCache) {
    +      this.authenticationResponseCache = authenticationResponseCache;
    +   }
    +
    +   @Override
    +   public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) {
    +      boolean retry = false; // default
    +      try {
    +         switch (response.getStatusCode()) {
    +            case 401:
    +               // Do not retry on 401 from authentication request
    +               Multimap<String, String> headers = command.getCurrentRequest().getHeaders();
    +               if (headers != null && headers.containsKey(AuthHeaders.AUTH_USER)
    +                        && headers.containsKey(AuthHeaders.AUTH_KEY) &&
!headers.containsKey(AuthHeaders.AUTH_TOKEN)) {
    +                  retry = false;
    +               } else {
    +                  closeClientButKeepContentStream(response);
    +                  authenticationResponseCache.invalidateAll();
    +                  retry = true;
    +                  
    +                  // always retry. not all swift servers say 'lease renew', e.g. softlayer
    +                  
    +//                  byte[] content = closeClientButKeepContentStream(response);
    +//                  if (content != null && new String(content).contains("lease
renew")) {
    +//                     logger.debug("invalidating authentication token");
    +//                     authenticationResponseCache.invalidateAll();
    +//                     retry = true;
    +//                  } else {
    +//                     retry = false;
    +//                  }
    +               }
    +               break;
    +         }
    +         return retry;
    +
    +      } finally {
    +         releasePayload(response);
    +      }
    +   }
    +
    +   /** 
    +    * Intercepts calls to the *other* RetryOnRenew instance, and uses the one above.
    +    * It's messy, but the only way I could find in the maze of guice. */
    +   public static class InterceptRetryOnRenewModule extends AbstractModule {
    +       AlwaysRetryOnRenew intereceptingRetryOnRenew;
    +
    +       public void inject(Injector injector) {
    +           intereceptingRetryOnRenew = injector.getInstance(AlwaysRetryOnRenew.class);
    +       }
    +       
    +       @Override
    +       protected void configure() {
    +           Matcher<? super Class<?>> classMatcher = Matchers.subclassesOf(RetryOnRenew.class);
    +           Matcher<? super Method> methodMatcher = new AbstractMatcher<Method>()
{
    --- End diff --
    
    Wow, nice workaround!


> Persistence to object store fails with 401 unauthorized after some time
> -----------------------------------------------------------------------
>
>                 Key: BROOKLYN-6
>                 URL: https://issues.apache.org/jira/browse/BROOKLYN-6
>             Project: Brooklyn
>          Issue Type: Bug
>         Environment: Softlayer
>            Reporter: Alex Heneveld
>            Priority: Critical
>
> When running with persistence to softlayer, Brooklyn works for some time (several days?)
but then gets a 401 unauthorized from softlayer.
> I suspect jclouds is not renewing the token, but this is TBC.
> Additionally, when this happens it affects standbys also so all nodes go into failed
status with master unknown and calls to `/v1/server/highAvailability` return a 500 making
all consoles unusable.
> Stack trace is:
> 2014-06-23 11:38:36,142 ERROR b.m.h.HighAvailabilityManagerImpl [brooklyn-execmanager-GGyYGAv5-599065]:
Problem in HA-poller: org.jclouds.rest.AuthorizationException: request: HEAD https://dal05.objectstorage.service.networklayer.com/v1/AUTH_1234_removed/brooklyn-plane/nodes/GGyYGAv5
HTTP/1.1 failed with response: HTTP/1.1 401 Unauthorized
> org.jclouds.rest.AuthorizationException: request: HEAD https://dal05.objectstorage.service.networklayer.com/v1/AUTH_1234_removed/brooklyn-plane/nodes/GGyYGAv5
HTTP/1.1 failed with response: HTTP/1.1 401 Unauthorized
> 	at org.jclouds.openstack.swift.handlers.ParseSwiftErrorFromHttpResponse.handleError(ParseSwiftErrorFromHttpResponse.java:61)
~[swift-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.http.handlers.DelegatingErrorHandler.handleError(DelegatingErrorHandler.java:67)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.http.internal.BaseHttpCommandExecutorService.shouldContinue(BaseHttpCommandExecutorService.java:180)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.http.internal.BaseHttpCommandExecutorService.invoke(BaseHttpCommandExecutorService.java:150)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.rest.internal.InvokeSyncToAsyncHttpMethod.invoke(InvokeSyncToAsyncHttpMethod.java:129)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.rest.internal.InvokeSyncToAsyncHttpMethod.apply(InvokeSyncToAsyncHttpMethod.java:95)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.rest.internal.InvokeSyncToAsyncHttpMethod.apply(InvokeSyncToAsyncHttpMethod.java:56)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.rest.internal.DelegatesToInvocationFunction.handle(DelegatesToInvocationFunction.java:156)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at org.jclouds.rest.internal.DelegatesToInvocationFunction.invoke(DelegatesToInvocationFunction.java:123)
~[jclouds-core-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at com.sun.proxy.$Proxy61.objectExists(Unknown Source) ~[na:na]
> 	at org.jclouds.openstack.swift.blobstore.SwiftBlobStore.blobExists(SwiftBlobStore.java:163)
~[swift-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	at brooklyn.entity.rebind.persister.jclouds.JcloudsStoreObjectAccessor.exists(JcloudsStoreObjectAccessor.java:33)
~[brooklyn-locations-jclouds-0.7.0-SNAPSHOT.jar:na]
> 	at brooklyn.entity.rebind.persister.StoreObjectAccessorLocking.exists(StoreObjectAccessorLocking.java:59)
~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at brooklyn.management.ha.ManagementPlaneSyncRecordPersisterToObjectStore.persist(ManagementPlaneSyncRecordPersisterToObjectStore.java:251)
~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at brooklyn.management.ha.ManagementPlaneSyncRecordPersisterToObjectStore.delta(ManagementPlaneSyncRecordPersisterToObjectStore.java:206)
~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at brooklyn.management.ha.HighAvailabilityManagerImpl.publishHealth(HighAvailabilityManagerImpl.java:282)
~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at brooklyn.management.ha.HighAvailabilityManagerImpl.publishAndCheck(HighAvailabilityManagerImpl.java:270)
~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at brooklyn.management.ha.HighAvailabilityManagerImpl$2.run(HighAvailabilityManagerImpl.java:245)
~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.6.0_27]
> 	at brooklyn.util.task.BasicExecutionManager$2$1.call(BasicExecutionManager.java:300)
~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at brooklyn.util.task.BasicExecutionManager$3.call(BasicExecutionManager.java:353) ~[brooklyn-core-0.7.0-SNAPSHOT.jar:na]
> 	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) ~[na:1.6.0_27]
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:166) ~[na:1.6.0_27]
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146) ~[na:1.6.0_27]
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.6.0_27]
> 	at java.lang.Thread.run(Thread.java:701) ~[na:1.6.0_27]
> Caused by: org.jclouds.http.HttpResponseException: request: HEAD https://dal05.objectstorage.service.networklayer.com/v1/AUTH_1234_removed/brooklyn-plane/nodes/GGyYGAv5
HTTP/1.1 failed with response: HTTP/1.1 401 Unauthorized
> 	at org.jclouds.openstack.swift.handlers.ParseSwiftErrorFromHttpResponse.handleError(ParseSwiftErrorFromHttpResponse.java:55)
~[swift-1.7.4-ea-brooklyn.1.jar:1.7.4-ea-brooklyn.1]
> 	... 25 common frames omitted



--
This message was sent by Atlassian JIRA
(v6.2#6252)

Mime
View raw message