brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From henev...@apache.org
Subject [08/21] incubator-brooklyn git commit: Make metrics on rebind, persistence, and HA available through REST API.
Date Sat, 15 Nov 2014 00:05:17 GMT
Make metrics on rebind, persistence, and HA available through REST API.

So that a management plane or human can determine server health and historic problems.


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/b9c1b6fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/b9c1b6fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/b9c1b6fc

Branch: refs/heads/master
Commit: b9c1b6fca99165f62b712ec7fd7967f7e4b4e283
Parents: 9dd1a95
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Thu Nov 13 03:42:54 2014 +0000
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Thu Nov 13 23:38:53 2014 +0000

----------------------------------------------------------------------
 .../entity/rebind/RebindExceptionHandler.java   |   6 +
 .../brooklyn/entity/rebind/RebindManager.java   |   5 +
 .../management/ha/HighAvailabilityManager.java  |   6 +
 .../rebind/PeriodicDeltaChangeListener.java     |  23 +--
 .../rebind/RebindExceptionHandlerImpl.java      |  54 +++++--
 .../entity/rebind/RebindManagerImpl.java        |  53 ++++++-
 .../persister/BrooklynPersistenceUtils.java     |   1 -
 .../persister/PersistenceActivityMetrics.java   |  83 ++++++++++
 .../ha/HighAvailabilityManagerImpl.java         | 155 ++++++++++++++-----
 .../NonDeploymentManagementContext.java         |   9 ++
 .../main/java/brooklyn/rest/api/ServerApi.java  |   7 +
 .../brooklyn/rest/resources/ServerResource.java |   7 +
 12 files changed, 346 insertions(+), 63 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/api/src/main/java/brooklyn/entity/rebind/RebindExceptionHandler.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/entity/rebind/RebindExceptionHandler.java b/api/src/main/java/brooklyn/entity/rebind/RebindExceptionHandler.java
index 61bcdaa..ce0af83 100644
--- a/api/src/main/java/brooklyn/entity/rebind/RebindExceptionHandler.java
+++ b/api/src/main/java/brooklyn/entity/rebind/RebindExceptionHandler.java
@@ -18,6 +18,8 @@
  */
 package brooklyn.entity.rebind;
 
+import java.util.List;
+
 import brooklyn.basic.BrooklynObject;
 import brooklyn.catalog.CatalogItem;
 import brooklyn.entity.Entity;
@@ -100,4 +102,8 @@ public interface RebindExceptionHandler {
     
     /** invoked after the complete rebind pass, always on success and possibly on failure */
     void onDone();
+    
+    List<Exception> getExceptions();
+    List<String> getWarnings();
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/entity/rebind/RebindManager.java b/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
index fcd71bc..8f34c5a 100644
--- a/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
+++ b/api/src/main/java/brooklyn/entity/rebind/RebindManager.java
@@ -19,6 +19,7 @@
 package brooklyn.entity.rebind;
 
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
@@ -124,4 +125,8 @@ public interface RebindManager {
      * setting if the process fails after the clear!) */
     @VisibleForTesting
     public void forcePersistNow(boolean full, @Nullable PersistenceExceptionHandler exceptionHandler);
+
+    /** Metrics about rebind, last success, etc. */
+    public Map<String,Object> getMetrics();
+    
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java b/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
index de37123..597e407 100644
--- a/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
+++ b/api/src/main/java/brooklyn/management/ha/HighAvailabilityManager.java
@@ -18,6 +18,8 @@
  */
 package brooklyn.management.ha;
 
+import java.util.Map;
+
 import com.google.common.annotations.Beta;
 import com.google.common.annotations.VisibleForTesting;
 
@@ -118,4 +120,8 @@ public interface HighAvailabilityManager {
     
     @VisibleForTesting
     ManagementPlaneSyncRecordPersister getPersister();
+
+    /** Returns a collection of metrics */
+    Map<String,Object> getMetrics();
+    
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/core/src/main/java/brooklyn/entity/rebind/PeriodicDeltaChangeListener.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/PeriodicDeltaChangeListener.java b/core/src/main/java/brooklyn/entity/rebind/PeriodicDeltaChangeListener.java
index 08ca53c..da35459 100644
--- a/core/src/main/java/brooklyn/entity/rebind/PeriodicDeltaChangeListener.java
+++ b/core/src/main/java/brooklyn/entity/rebind/PeriodicDeltaChangeListener.java
@@ -38,10 +38,10 @@ import brooklyn.entity.Feed;
 import brooklyn.entity.basic.BrooklynTaskTags;
 import brooklyn.entity.basic.EntityInternal;
 import brooklyn.entity.rebind.persister.BrooklynPersistenceUtils;
+import brooklyn.entity.rebind.persister.PersistenceActivityMetrics;
 import brooklyn.internal.BrooklynFeatureEnablement;
 import brooklyn.location.Location;
 import brooklyn.management.ExecutionContext;
-import brooklyn.management.ExecutionManager;
 import brooklyn.management.Task;
 import brooklyn.mementos.BrooklynMementoPersister;
 import brooklyn.policy.Enricher;
@@ -50,7 +50,6 @@ import brooklyn.util.collections.MutableMap;
 import brooklyn.util.collections.MutableSet;
 import brooklyn.util.exceptions.Exceptions;
 import brooklyn.util.exceptions.RuntimeInterruptedException;
-import brooklyn.util.task.BasicExecutionContext;
 import brooklyn.util.task.ScheduledTask;
 import brooklyn.util.task.Tasks;
 import brooklyn.util.time.CountdownTimer;
@@ -59,6 +58,7 @@ import brooklyn.util.time.Time;
 
 import com.google.api.client.util.Lists;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 
@@ -178,17 +178,14 @@ public class PeriodicDeltaChangeListener implements ChangeListener {
     
     private final Semaphore persistingMutex = new Semaphore(1);
     private final Object startMutex = new Object();
+
+    private PersistenceActivityMetrics metrics;
     
-    /** @deprecated since 0.7.0 pass in an {@link ExecutionContext} and a {@link Duration} */
-    @Deprecated
-    public PeriodicDeltaChangeListener(ExecutionManager executionManager, BrooklynMementoPersister persister, PersistenceExceptionHandler exceptionHandler, long periodMillis) {
-        this(new BasicExecutionContext(executionManager), persister, exceptionHandler, Duration.of(periodMillis, TimeUnit.MILLISECONDS));
-    }
-    
-    public PeriodicDeltaChangeListener(ExecutionContext executionContext, BrooklynMementoPersister persister, PersistenceExceptionHandler exceptionHandler, Duration period) {
+    public PeriodicDeltaChangeListener(ExecutionContext executionContext, BrooklynMementoPersister persister, PersistenceExceptionHandler exceptionHandler, PersistenceActivityMetrics metrics, Duration period) {
         this.executionContext = executionContext;
         this.persister = persister;
         this.exceptionHandler = exceptionHandler;
+        this.metrics = metrics;
         this.period = period;
         
         this.persistPoliciesEnabled = BrooklynFeatureEnablement.isEnabled(BrooklynFeatureEnablement.FEATURE_POLICY_PERSISTENCE_PROPERTY);
@@ -210,20 +207,28 @@ public class PeriodicDeltaChangeListener implements ChangeListener {
                 @Override public Task<Void> call() {
                     return Tasks.<Void>builder().dynamic(false).name("periodic-persister").body(new Callable<Void>() {
                         public Void call() {
+                            Stopwatch timer = Stopwatch.createStarted();
                             try {
                                 persistNow();
+                                metrics.noteSuccess(Duration.of(timer));
                                 return null;
                             } catch (RuntimeInterruptedException e) {
                                 LOG.debug("Interrupted persisting change-delta (rethrowing)", e);
+                                metrics.noteFailure(Duration.of(timer));
+                                metrics.noteError(e.toString());
                                 Thread.currentThread().interrupt();
                                 return null;
                             } catch (Exception e) {
                                 // Don't rethrow: the behaviour of executionManager is different from a scheduledExecutorService,
                                 // if we throw an exception, then our task will never get executed again
                                 LOG.error("Problem persisting change-delta", e);
+                                metrics.noteFailure(Duration.of(timer));
+                                metrics.noteError(e.toString());
                                 return null;
                             } catch (Throwable t) {
                                 LOG.warn("Problem persisting change-delta (rethrowing)", t);
+                                metrics.noteFailure(Duration.of(timer));
+                                metrics.noteError(t.toString());
                                 throw Exceptions.propagate(t);
                             }
                         }}).build();

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/core/src/main/java/brooklyn/entity/rebind/RebindExceptionHandlerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/RebindExceptionHandlerImpl.java b/core/src/main/java/brooklyn/entity/rebind/RebindExceptionHandlerImpl.java
index ac41c8b..2b61173 100644
--- a/core/src/main/java/brooklyn/entity/rebind/RebindExceptionHandlerImpl.java
+++ b/core/src/main/java/brooklyn/entity/rebind/RebindExceptionHandlerImpl.java
@@ -41,6 +41,7 @@ import brooklyn.util.collections.QuorumCheck;
 import brooklyn.util.exceptions.Exceptions;
 import brooklyn.util.text.Strings;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 
@@ -62,9 +63,12 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
     protected final Set<String> missingFeeds = Sets.newConcurrentHashSet();
     protected final Set<String> missingCatalogItems = Sets.newConcurrentHashSet();
     protected final Set<String> creationFailedIds = Sets.newConcurrentHashSet();
+    
     protected final Set<Exception> addPolicyFailures = Sets.newConcurrentHashSet();
     protected final Set<Exception> loadPolicyFailures = Sets.newConcurrentHashSet();
-    protected final List<Exception> exceptions = Collections.synchronizedList(Lists.<Exception>newArrayList());
+    
+    protected final Set<String> warnings = Collections.synchronizedSet(Sets.<String>newLinkedHashSet());
+    protected final Set<Exception> exceptions = Collections.synchronizedSet(Sets.<Exception>newLinkedHashSet());
     
     protected RebindContext context;
     protected boolean started = false;
@@ -113,6 +117,15 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
         this.loadPolicyFailureMode = checkNotNull(builder.deserializePolicyFailureMode, "deserializePolicyFailureMode");
         this.danglingRefsQuorumRequiredHealthy = checkNotNull(builder.danglingRefsQuorumRequiredHealthy, "danglingRefsQuorumRequiredHealthy");
     }
+    
+    protected void warn(String message) {
+        warn(message, null);
+    }
+    protected void warn(String message, Throwable optionalError) {
+        if (optionalError==null) LOG.warn(message);
+        else LOG.warn(message, optionalError);
+        warnings.add(message);
+    }
 
     @Override
     public void onStart(RebindContext context) {
@@ -142,7 +155,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
                         loadPolicyFailures.add(new IllegalStateException(errmsg, e));
                         break;
                     case CONTINUE:
-                        LOG.warn(errmsg+"; continuing", e);
+                        warn(errmsg+"; continuing: "+e, e);
                         break;
                     default:
                         throw new IllegalStateException("Unexpected state '"+loadPolicyFailureMode+"' for loadPolicyFailureMode");
@@ -160,7 +173,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
         if (danglingRefFailureMode == RebindManager.RebindFailureMode.FAIL_FAST) {
             throw new IllegalStateException("No entity found with id "+id);
         } else {
-            LOG.warn("No entity found with id "+id+"; returning null");
+            warn("No entity found with id "+id+"; returning null");
             return null;
         }
     }
@@ -171,7 +184,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
         if (danglingRefFailureMode == RebindManager.RebindFailureMode.FAIL_FAST) {
             throw new IllegalStateException("No location found with id "+id);
         } else {
-            LOG.warn("No location found with id "+id+"; returning null");
+            warn("No location found with id "+id+"; returning null");
             return null;
         }
     }
@@ -182,7 +195,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
         if (danglingRefFailureMode == RebindManager.RebindFailureMode.FAIL_FAST) {
             throw new IllegalStateException("No policy found with id "+id);
         } else {
-            LOG.warn("No policy found with id "+id+"; returning null");
+            warn("No policy found with id "+id+"; returning null");
             return null;
         }
     }
@@ -193,7 +206,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
         if (danglingRefFailureMode == RebindManager.RebindFailureMode.FAIL_FAST) {
             throw new IllegalStateException("No enricher found with id "+id);
         } else {
-            LOG.warn("No enricher found with id "+id+"; returning null");
+            warn("No enricher found with id "+id+"; returning null");
             return null;
         }
     }
@@ -204,7 +217,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
         if (danglingRefFailureMode == RebindManager.RebindFailureMode.FAIL_FAST) {
             throw new IllegalStateException("No feed found with id "+id);
         } else {
-            LOG.warn("No feed found with id "+id+"; returning null");
+            warn("No feed found with id "+id+"; returning null");
             return null;
         }
     }
@@ -215,7 +228,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
         if (danglingRefFailureMode == RebindManager.RebindFailureMode.FAIL_FAST) {
             throw new IllegalStateException("No catalog item found with id "+id);
         } else {
-            LOG.warn("No catalog item found with id "+id+"; returning null");
+            warn("No catalog item found with id "+id+"; returning null");
             return null;
         }
     }
@@ -256,7 +269,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
                 addPolicyFailures.add(new IllegalStateException(errmsg, e));
                 break;
             case CONTINUE:
-                LOG.warn(errmsg+"; continuing", e);
+                warn(errmsg+"; continuing", e);
                 creationFailedIds.add(instance.getId());
                 break;
             default:
@@ -282,7 +295,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
             addPolicyFailures.add(new IllegalStateException(errmsg, e));
             break;
         case CONTINUE:
-            LOG.warn(errmsg+"; continuing", e);
+            warn(errmsg+"; continuing", e);
             break;
         default:
             throw new IllegalStateException("Unexpected state '"+addPolicyFailureMode+"' for addPolicyFailureMode");
@@ -301,7 +314,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
             addPolicyFailures.add(new IllegalStateException(errmsg, e));
             break;
         case CONTINUE:
-            LOG.warn(errmsg+"; continuing", e);
+            warn(errmsg+"; continuing", e);
             break;
         default:
             throw new IllegalStateException("Unexpected state '"+addPolicyFailureMode+"' for addPolicyFailureMode");
@@ -320,7 +333,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
             addPolicyFailures.add(new IllegalStateException(errmsg, e));
             break;
         case CONTINUE:
-            LOG.warn(errmsg+"; continuing", e);
+            warn(errmsg+"; continuing", e);
             break;
         default:
             throw new IllegalStateException("Unexpected state '"+addPolicyFailureMode+"' for addPolicyFailureMode");
@@ -349,7 +362,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
                     LOG.debug("Rebind: while interrupted, received "+errmsg+"/"+e+"; throwing interruption", e);
                 throw Exceptions.propagate(new InterruptedException("Detected interruptiong while not sleeping, due to secondary error rebinding: "+errmsg+"/"+e));
             }
-            LOG.warn("Rebind: continuing after "+errmsg, e);
+            warn("Rebind: continuing after "+errmsg, e);
         }
     }
     
@@ -364,6 +377,8 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
             throw Exceptions.propagate(e);
         
         onDoneImpl(e);
+        exceptions.add(e);
+        
         throw new IllegalStateException("Rebind failed", e); // should have thrown exception above
     }
     
@@ -386,7 +401,7 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
                 allExceptions.add(new IllegalStateException("Dangling references ("+totalDangling+" of "+totalItems+") present without rebind context"));
             } else {
                 if (!danglingRefsQuorumRequiredHealthy.isQuorate(totalFound, totalItems)) {
-                    LOG.warn("Dangling item"+Strings.s(totalDangling)+" ("+totalDangling+" of "+totalItems+") found on rebind exceeds quorum, assuming failed: "+danglingIds);
+                    warn("Dangling item"+Strings.s(totalDangling)+" ("+totalDangling+" of "+totalItems+") found on rebind exceeds quorum, assuming failed: "+danglingIds);
                     allExceptions.add(new IllegalStateException("Too many dangling references: "+totalDangling+" of "+totalItems));
                 } else {
                     LOG.info("Dangling item"+Strings.s(totalDangling)+" ("+totalDangling+" of "+totalItems+") found on rebind, assuming deleted: "+danglingIds);
@@ -439,4 +454,15 @@ public class RebindExceptionHandlerImpl implements RebindExceptionHandler {
             throw compoundException;
         }
     }
+    
+    @Override
+    public List<Exception> getExceptions() {
+        return ImmutableList.copyOf(exceptions);
+    }
+    
+    @Override
+    public List<String> getWarnings() {
+        return ImmutableList.copyOf(warnings);
+    }
+    
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java b/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
index 4252a25..e24a818 100644
--- a/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
+++ b/core/src/main/java/brooklyn/entity/rebind/RebindManagerImpl.java
@@ -58,6 +58,7 @@ import brooklyn.entity.proxying.InternalLocationFactory;
 import brooklyn.entity.proxying.InternalPolicyFactory;
 import brooklyn.entity.rebind.persister.BrooklynMementoPersisterToObjectStore;
 import brooklyn.entity.rebind.persister.BrooklynPersistenceUtils;
+import brooklyn.entity.rebind.persister.PersistenceActivityMetrics;
 import brooklyn.event.feed.AbstractFeed;
 import brooklyn.internal.BrooklynFeatureEnablement;
 import brooklyn.location.Location;
@@ -90,6 +91,7 @@ import brooklyn.mementos.TreeNode;
 import brooklyn.policy.Enricher;
 import brooklyn.policy.Policy;
 import brooklyn.policy.basic.AbstractPolicy;
+import brooklyn.util.collections.MutableList;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.collections.QuorumCheck;
 import brooklyn.util.collections.QuorumCheck.QuorumChecks;
@@ -174,7 +176,12 @@ public class RebindManagerImpl implements RebindManager {
     private RebindFailureMode addPolicyFailureMode;
     private RebindFailureMode loadPolicyFailureMode;
     private QuorumCheck danglingRefsQuorumRequiredHealthy;
+    
+    private PersistenceActivityMetrics rebindMetrics = new PersistenceActivityMetrics();
+    private PersistenceActivityMetrics persistMetrics = new PersistenceActivityMetrics();
 
+    Integer firstRebindAppCount, firstRebindEntityCount, firstRebindItemCount;
+    
     /**
      * For tracking if rebinding, for {@link AbstractEnricher#isRebinding()} etc.
      *  
@@ -256,7 +263,7 @@ public class RebindManagerImpl implements RebindManager {
         }
         this.persistenceStoreAccess = checkNotNull(val, "persister");
         
-        this.persistenceRealChangeListener = new PeriodicDeltaChangeListener(managementContext.getServerExecutionContext(), persistenceStoreAccess, exceptionHandler, periodicPersistPeriod);
+        this.persistenceRealChangeListener = new PeriodicDeltaChangeListener(managementContext.getServerExecutionContext(), persistenceStoreAccess, exceptionHandler, persistMetrics, periodicPersistPeriod);
         this.persistencePublicChangeListener = new SafeChangeListener(persistenceRealChangeListener);
         
         if (persistenceRunning) {
@@ -530,8 +537,8 @@ public class RebindManagerImpl implements RebindManager {
             rebindActive.acquire();
         } catch (InterruptedException e1) { Exceptions.propagate(e1); }
         RebindTracker.setRebinding();
+        Stopwatch timer = Stopwatch.createStarted();
         try {
-            Stopwatch timer = Stopwatch.createStarted();
             Reflections reflections = new Reflections(classLoader);
             RebindContextImpl rebindContext = new RebindContextImpl(exceptionHandler, classLoader);
             
@@ -970,6 +977,14 @@ public class RebindManagerImpl implements RebindManager {
             }
 
             exceptionHandler.onDone();
+            
+            rebindMetrics.noteSuccess(Duration.of(timer));
+            noteErrors(exceptionHandler, null);
+            if (firstRebindAppCount==null) {
+                firstRebindAppCount = apps.size();
+                firstRebindEntityCount = rebindContext.getEntities().size();
+                firstRebindItemCount = rebindContext.getAllBrooklynObjects().size();
+            }
 
             if (!isEmpty) {
                 BrooklynLogging.log(LOG, shouldLogRebinding() ? LoggingLevel.INFO : LoggingLevel.DEBUG, 
@@ -990,13 +1005,30 @@ public class RebindManagerImpl implements RebindManager {
             return apps;
 
         } catch (Exception e) {
+            rebindMetrics.noteFailure(Duration.of(timer));
+            
+            Exceptions.propagateIfFatal(e);
+            noteErrors(exceptionHandler, e);
             throw exceptionHandler.onFailed(e);
+            
         } finally {
             rebindActive.release();
             RebindTracker.reset();
         }
     }
 
+    private void noteErrors(final RebindExceptionHandler exceptionHandler, Exception primaryException) {
+        List<Exception> exceptions = exceptionHandler.getExceptions();
+        List<String> warnings = exceptionHandler.getWarnings();
+        if (primaryException!=null || !exceptions.isEmpty() || !warnings.isEmpty()) {
+            List<String> messages = MutableList.<String>of();
+            if (primaryException!=null) messages.add(primaryException.toString());
+            for (Exception e: exceptions) messages.add(e.toString());
+            for (String w: warnings) messages.add(w);
+            rebindMetrics.noteError(messages);
+        }
+    }
+    
     private String findCatalogItemId(ClassLoader cl, Map<String, EntityMementoManifest> entityIdToManifest, EntityMementoManifest entityManifest) {
         if (entityManifest.getCatalogItemId() != null) {
             return entityManifest.getCatalogItemId();
@@ -1418,7 +1450,24 @@ public class RebindManagerImpl implements RebindManager {
     }
 
     @Override
+    public Map<String, Object> getMetrics() {
+        Map<String,Object> result = MutableMap.of();
+
+        result.put("rebind", rebindMetrics.asMap());
+        result.put("persist", persistMetrics.asMap());
+        
+        // include first rebind counts, so we know whether we rebinded or not
+        result.put("firstRebindCounts", MutableMap.of(
+            "applications", firstRebindAppCount,
+            "entities", firstRebindEntityCount,
+            "allItems", firstRebindItemCount));
+        
+        return result;
+    }
+
+    @Override
     public String toString() {
         return super.toString()+"[mgmt="+managementContext.getManagementNodeId()+"]";
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/core/src/main/java/brooklyn/entity/rebind/persister/BrooklynPersistenceUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/persister/BrooklynPersistenceUtils.java b/core/src/main/java/brooklyn/entity/rebind/persister/BrooklynPersistenceUtils.java
index 54990b3..8b5317d 100644
--- a/core/src/main/java/brooklyn/entity/rebind/persister/BrooklynPersistenceUtils.java
+++ b/core/src/main/java/brooklyn/entity/rebind/persister/BrooklynPersistenceUtils.java
@@ -29,7 +29,6 @@ import brooklyn.config.BrooklynServerConfig;
 import brooklyn.config.BrooklynServerPaths;
 import brooklyn.entity.Entity;
 import brooklyn.entity.Feed;
-import brooklyn.entity.basic.AbstractEntity;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.EntityInternal;
 import brooklyn.entity.rebind.BrooklynObjectType;

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/core/src/main/java/brooklyn/entity/rebind/persister/PersistenceActivityMetrics.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/persister/PersistenceActivityMetrics.java b/core/src/main/java/brooklyn/entity/rebind/persister/PersistenceActivityMetrics.java
new file mode 100644
index 0000000..8315156
--- /dev/null
+++ b/core/src/main/java/brooklyn/entity/rebind/persister/PersistenceActivityMetrics.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package brooklyn.entity.rebind.persister;
+
+import java.util.List;
+import java.util.Map;
+
+import brooklyn.util.collections.MutableList;
+import brooklyn.util.collections.MutableMap;
+import brooklyn.util.time.Duration;
+
+public class PersistenceActivityMetrics {
+    
+    final static int MAX_ERRORS = 200;
+    
+    long count=0, failureCount=0;
+    Long lastSuccessTime, lastDuration, lastFailureTime;
+    List<Map<String,Object>> errorMessages = MutableList.of();
+
+    public void noteSuccess(Duration duration) {
+        count++;
+        lastSuccessTime = System.currentTimeMillis();
+        lastDuration = duration.toMilliseconds();
+    }
+    
+    public void noteFailure(Duration duration) {
+        count++;
+        failureCount++;
+        lastFailureTime = System.currentTimeMillis();
+        lastDuration = duration.toMilliseconds();
+    }
+
+    public void noteError(String error) {
+        noteErrorObject(error);
+    }
+    
+    public void noteError(List<?> error) {
+        noteErrorObject(error);
+    }
+    
+    /** error should be json-serializable; exceptions can be problematic */
+    protected synchronized void noteErrorObject(Object error) {
+        errorMessages.add(0, MutableMap.<String,Object>of("error", error, "timestamp", System.currentTimeMillis()));
+        while (errorMessages.size() > MAX_ERRORS) {
+            errorMessages.remove(errorMessages.size()-1);
+        }
+    }
+    
+    public synchronized Map<String,Object> asMap() {
+        Map<String,Object> result = MutableMap.of();
+        result.put("count", count);
+        result.put("lastSuccessTimeUtc", lastSuccessTime);
+        result.put("lastSuccessTimeMillisSince", since(lastSuccessTime));
+        result.put("lastDuration", lastDuration);
+        result.put("failureCount", failureCount);
+        result.put("lastFailureTimeUtc", lastFailureTime);
+        result.put("lastFailureTimeMillisSince", since(lastFailureTime));
+        result.put("errorMessages", MutableList.copyOf(errorMessages));
+        return result;
+    }
+
+    private Long since(Long time) {
+        if (time==null) return null;
+        return System.currentTimeMillis() - time;
+    }
+    
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
index 62b06c0..73d431d 100644
--- a/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
+++ b/core/src/main/java/brooklyn/management/ha/HighAvailabilityManagerImpl.java
@@ -23,6 +23,8 @@ import static com.google.common.base.Preconditions.checkState;
 
 import java.io.IOException;
 import java.net.URI;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.Callable;
 
 import javax.annotation.Nullable;
@@ -42,6 +44,7 @@ import brooklyn.entity.basic.ConfigKeys;
 import brooklyn.entity.basic.EntityInternal;
 import brooklyn.entity.rebind.RebindManager;
 import brooklyn.entity.rebind.persister.BrooklynPersistenceUtils;
+import brooklyn.entity.rebind.persister.PersistenceActivityMetrics;
 import brooklyn.entity.rebind.plane.dto.BasicManagementNodeSyncRecord;
 import brooklyn.entity.rebind.plane.dto.ManagementPlaneSyncRecordImpl;
 import brooklyn.entity.rebind.plane.dto.ManagementPlaneSyncRecordImpl.Builder;
@@ -54,6 +57,7 @@ import brooklyn.management.internal.LocalEntityManager;
 import brooklyn.management.internal.LocationManagerInternal;
 import brooklyn.management.internal.ManagementContextInternal;
 import brooklyn.management.internal.ManagementTransitionInfo.ManagementTransitionMode;
+import brooklyn.util.collections.MutableList;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.exceptions.Exceptions;
 import brooklyn.util.task.ScheduledTask;
@@ -66,6 +70,7 @@ import com.google.common.annotations.Beta;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
 import com.google.common.base.Ticker;
 import com.google.common.collect.Iterables;
 
@@ -132,13 +137,21 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
     private volatile boolean nodeStateTransitionComplete = false;
     private volatile long priority = 0;
     
+    private final static int MAX_NODE_STATE_HISTORY = 200;
+    private final List<Map<String,Object>> nodeStateHistory = MutableList.of();
+    
     private volatile transient Duration pollPeriodLocalOverride;
     private volatile transient Duration heartbeatTimeoutOverride;
 
     private volatile ManagementPlaneSyncRecord lastSyncRecord;
     
+    private volatile PersistenceActivityMetrics managementStateWritePersistenceMetrics = new PersistenceActivityMetrics();
+    private volatile PersistenceActivityMetrics managementStateReadPersistenceMetrics = new PersistenceActivityMetrics();
+    private final long startTimeUtc;
+    
     public HighAvailabilityManagerImpl(ManagementContextInternal managementContext) {
         this.managementContext = managementContext;
+        startTimeUtc = localTickerUtc.read();
     }
 
     @Override
@@ -228,9 +241,9 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
     @Override
     public void start(HighAvailabilityMode startMode) {
         nodeStateTransitionComplete = true;
-        // always start in standby; it may get promoted to master or hot_standby in this method
+        // always start in standby, unless hot backup; it may get promoted to master or hot_standby in this method
         // (depending on startMode; but for startMode STANDBY or HOT_STANDBY it will not promote until the next election)
-        nodeState = ManagementNodeState.STANDBY;
+        setInternalNodeState(startMode==HighAvailabilityMode.HOT_BACKUP ? ManagementNodeState.HOT_BACKUP : ManagementNodeState.STANDBY);
         disabled = false;
         running = true;
         changeMode(startMode, true, true);
@@ -246,13 +259,13 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
     public void changeMode(HighAvailabilityMode startMode, boolean preventElectionOnExplicitStandbyMode, boolean failOnExplicitStandbyModeIfNoMaster) {
         if (!running) {
             // if was not running then start as disabled mode, then proceed as normal
-            LOG.info("HA changing mode to "+startMode+" from "+nodeState+" when not running, forcing an intermediate start as DISABLED then will convert to "+startMode);
+            LOG.info("HA changing mode to "+startMode+" from "+getInternalNodeState()+" when not running, forcing an intermediate start as DISABLED then will convert to "+startMode);
             start(HighAvailabilityMode.DISABLED);
         }
         if (getNodeState()==ManagementNodeState.FAILED || getNodeState()==ManagementNodeState.INITIALIZING) {
             if (startMode!=HighAvailabilityMode.DISABLED) {
                 // if coming from FAILED (or INITIALIZING because we skipped start call) then treat as cold standby
-                nodeState = ManagementNodeState.STANDBY; 
+                setInternalNodeState(ManagementNodeState.STANDBY);
             }
         }
         
@@ -272,7 +285,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
             case MASTER:
             case AUTO:
             case DISABLED:
-                // no action needed, will do anything necessary below
+                // no action needed, will do anything necessary below (or above)
                 break;
             case HOT_STANDBY: 
             case HOT_BACKUP: 
@@ -288,17 +301,18 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
         case AUTO:
             // don't care; let's start and see if we promote ourselves
             publishAndCheck(true);
-            switch (nodeState) {
+            switch (getInternalNodeState()) {
             case HOT_BACKUP:
-                if (!nodeStateTransitionComplete) throw new IllegalStateException("Cannot switch to AUTO when in the middle of a transition to "+nodeState);
-                // else change us to hot standby and continue to below
-                nodeState = ManagementNodeState.HOT_STANDBY;
+                if (!nodeStateTransitionComplete) throw new IllegalStateException("Cannot switch to AUTO when in the middle of a transition to "+getInternalNodeState());
+                // else change us to standby, desiring to go to hot standby, and continue to below
+                setInternalNodeState(ManagementNodeState.STANDBY);
+                startMode = HighAvailabilityMode.HOT_STANDBY;
             case HOT_STANDBY:
             case STANDBY:
                 ManagementPlaneSyncRecord newState = loadManagementPlaneSyncRecord(true);
                 String masterNodeId = newState.getMasterNodeId();
                 ManagementNodeSyncRecord masterNodeDetails = newState.getManagementNodes().get(masterNodeId);
-                LOG.info("Management node "+ownNodeId+" running as HA " + nodeState + " autodetected, " +
+                LOG.info("Management node "+ownNodeId+" running as HA " + getInternalNodeState() + " autodetected, " +
                     (Strings.isBlank(masterNodeId) ? "no master currently (other node should promote itself soon)" : "master "
                         + (existingMaster==null ? "(new) " : "")
                         + "is "+masterNodeId +
@@ -308,7 +322,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
                 LOG.info("Management node "+ownNodeId+" running as HA MASTER autodetected");
                 break;
             default:
-                throw new IllegalStateException("Management node "+ownNodeId+" set to HA AUTO, encountered unexpected mode "+nodeState);
+                throw new IllegalStateException("Management node "+ownNodeId+" set to HA AUTO, encountered unexpected mode "+getInternalNodeState());
             }
             break;
         case MASTER:
@@ -322,16 +336,23 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
                 LOG.info("Management node "+ownNodeId+" already running as HA MASTER, when set explicitly");
             }
             break;
+        case HOT_BACKUP:
         case STANDBY:
         case HOT_STANDBY:
-            if (!preventElectionOnExplicitStandbyMode)
-                publishAndCheck(true);
-            if (failOnExplicitStandbyModeIfNoMaster && existingMaster==null) {
-                LOG.error("Management node "+ownNodeId+" detected no master when "+startMode+" requested and existing master required; failing.");
-                throw new IllegalStateException("No existing master; cannot start as "+startMode);
+            if (getInternalNodeState()==ManagementNodeState.STANDBY || getInternalNodeState()==ManagementNodeState.HOT_STANDBY) {
+                if (!preventElectionOnExplicitStandbyMode)
+                    publishAndCheck(true);
+                if (failOnExplicitStandbyModeIfNoMaster && existingMaster==null) {
+                    LOG.error("Management node "+ownNodeId+" detected no master when "+startMode+" requested and existing master required; failing.");
+                    throw new IllegalStateException("No existing master; cannot start as "+startMode);
+                }
+            }
+            if (startMode==HighAvailabilityMode.HOT_BACKUP) {
+                setInternalNodeState(ManagementNodeState.HOT_BACKUP);
+            } else {
+                setInternalNodeState(ManagementNodeState.STANDBY);
+                // might jump to hot_standby next
             }
-            // continue to below (above lines skipped for hot backup)
-        case HOT_BACKUP:
             String message = "Management node "+ownNodeId+" running as HA "+getNodeState()+" (";
             if (getNodeState().toString().equals(startMode.toString()))
                 message += "explicitly requested";
@@ -355,7 +376,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
             break;
         case DISABLED:
             // safe just to run even if we weren't master
-            LOG.info("Management node "+ownNodeId+" HA DISABLED (was "+nodeState+")");
+            LOG.info("Management node "+ownNodeId+" HA DISABLED (was "+getInternalNodeState()+")");
             demoteTo(ManagementNodeState.FAILED);
             if (pollingTask!=null) pollingTask.cancel(true);
             break;
@@ -370,7 +391,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
                 startMode = HighAvailabilityMode.STANDBY;
             }
         }
-        if ((nodeState==ManagementNodeState.STANDBY && startMode==HighAvailabilityMode.HOT_STANDBY) || 
+        if ((getInternalNodeState()==ManagementNodeState.STANDBY && startMode==HighAvailabilityMode.HOT_STANDBY) || 
                 (startMode==HighAvailabilityMode.HOT_BACKUP)) {
             nodeStateTransitionComplete = false;
             if (startMode==HighAvailabilityMode.HOT_STANDBY) {
@@ -422,7 +443,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
         boolean wasRunning = running;
         
         running = false;
-        nodeState = newState;
+        setInternalNodeState(newState);
         if (pollingTask != null) pollingTask.cancel(true);
         
         if (wasRunning) {
@@ -437,22 +458,40 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
     
     /** returns the node state this node is trying to be in */
     public ManagementNodeState getTransitionTargetNodeState() {
+        return getInternalNodeState();
+    }
+    
+    protected ManagementNodeState getInternalNodeState() {
         return nodeState;
     }
     
+    protected void setInternalNodeState(ManagementNodeState newState) {
+        synchronized (nodeStateHistory) {
+            if (this.nodeState != newState) {
+                nodeStateHistory.add(0, MutableMap.<String,Object>of("state", newState, "timestamp", currentTimeMillis()));
+                while (nodeStateHistory.size()>MAX_NODE_STATE_HISTORY) {
+                    nodeStateHistory.remove(nodeStateHistory.size()-1);
+                }
+            }
+            
+            this.nodeState = newState;
+        }
+    }
+
     @SuppressWarnings("deprecation")
     @Override
     public ManagementNodeState getNodeState() {
-        if (nodeState==ManagementNodeState.FAILED) return nodeState;
+        ManagementNodeState myNodeState = getInternalNodeState();
+        if (myNodeState==ManagementNodeState.FAILED) return getInternalNodeState();
         // if target is master then we claim already being master, to prevent other nodes from taking it
         // (we may fail subsequently of course)
-        if (nodeState==ManagementNodeState.MASTER) return nodeState;
+        if (myNodeState==ManagementNodeState.MASTER) return myNodeState;
         
         // for backwards compatibility; remove in 0.8.0
-        if (nodeState==ManagementNodeState.UNINITIALISED) return ManagementNodeState.INITIALIZING;
+        if (myNodeState==ManagementNodeState.UNINITIALISED) return ManagementNodeState.INITIALIZING;
         
         if (!nodeStateTransitionComplete) return ManagementNodeState.INITIALIZING;
-        return nodeState;
+        return myNodeState;
     }
 
     public ManagementPlaneSyncRecord getLastManagementPlaneSyncRecord() {
@@ -516,10 +555,19 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
             return;
         }
         
-        ManagementNodeSyncRecord memento = createManagementNodeSyncRecord(false);
-        Delta delta = ManagementPlaneSyncRecordDeltaImpl.builder().node(memento).build();
-        persister.delta(delta);
-        if (LOG.isTraceEnabled()) LOG.trace("Published management-node health: {}", memento);
+        Stopwatch timer = Stopwatch.createStarted();
+        try {
+            ManagementNodeSyncRecord memento = createManagementNodeSyncRecord(false);
+            Delta delta = ManagementPlaneSyncRecordDeltaImpl.builder().node(memento).build();
+            persister.delta(delta);
+            managementStateWritePersistenceMetrics.noteSuccess(Duration.of(timer));
+            if (LOG.isTraceEnabled()) LOG.trace("Published management-node health: {}", memento);
+        } catch (Throwable t) {
+            managementStateWritePersistenceMetrics.noteFailure(Duration.of(timer));
+            managementStateWritePersistenceMetrics.noteError(t.toString());
+            LOG.debug("Error publishing management-node health (rethrowing): "+t);
+            throw Exceptions.propagate(t);
+        }
     }
     
     protected synchronized void publishDemotion(boolean demotingFromMaster) {
@@ -707,8 +755,8 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
                 LOG.warn("Problem in promption-listener (continuing)", e);
             }
         }
-        boolean wasHot = (nodeState==ManagementNodeState.HOT_STANDBY || nodeState==ManagementNodeState.HOT_BACKUP);
-        nodeState = ManagementNodeState.MASTER;
+        boolean wasHot = (getInternalNodeState()==ManagementNodeState.HOT_STANDBY || getInternalNodeState()==ManagementNodeState.HOT_BACKUP);
+        setInternalNodeState(ManagementNodeState.MASTER);
         publishPromotionToMaster();
         try {
             if (wasHot) {
@@ -717,7 +765,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
                 managementContext.getRebindManager().stopReadOnly();
                 clearManagedItems(ManagementTransitionMode.REBINDING_DESTROYED);
             }
-            managementContext.getRebindManager().rebind(managementContext.getCatalog().getRootClassLoader(), null, nodeState);
+            managementContext.getRebindManager().rebind(managementContext.getCatalog().getRootClassLoader(), null, getInternalNodeState());
         } catch (Exception e) {
             LOG.error("Management node enountered problem during rebind when promoting self to master; demoting to FAILED and rethrowing: "+e);
             demoteTo(ManagementNodeState.FAILED);
@@ -746,7 +794,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
             LOG.warn("Ignoring demote-from-master request, as HighAvailabilityManager is no longer running");
             return;
         }
-        boolean wasMaster = nodeState == ManagementNodeState.MASTER;
+        boolean wasMaster = (getInternalNodeState() == ManagementNodeState.MASTER);
         if (wasMaster) backupOnDemotionIfNeeded();
         ManagementTransitionMode mode = (wasMaster ? ManagementTransitionMode.REBINDING_NO_LONGER_PRIMARY : ManagementTransitionMode.REBINDING_DESTROYED);
 
@@ -756,9 +804,9 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
         case FAILED: 
         case HOT_BACKUP:
         case STANDBY:
-            nodeState = toState; break;
+            setInternalNodeState(toState); break;
         case HOT_STANDBY:
-            nodeState = ManagementNodeState.STANDBY; break;
+            setInternalNodeState(ManagementNodeState.STANDBY); break;
         default:
             throw new IllegalStateException("Illegal target state: "+toState);
         }
@@ -822,7 +870,7 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
     protected boolean attemptHotProxy(ManagementNodeState toState) {
         try {
             Preconditions.checkState(nodeStateTransitionComplete==false, "Must be in transitioning state to go into "+toState);
-            nodeState = toState;
+            setInternalNodeState(toState);
             managementContext.getRebindManager().startReadOnly(toState);
             
             return true;
@@ -859,6 +907,8 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
         
         int maxLoadAttempts = 5;
         Exception lastException = null;
+        Stopwatch timer = Stopwatch.createStarted();
+
         for (int i = 0; i < maxLoadAttempts; i++) {
             try {
                 ManagementPlaneSyncRecord result = persister.loadSyncRecord();
@@ -881,6 +931,11 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
                     }
                     result = builder.build();
                 }
+                
+                if (i>0) {
+                    managementStateReadPersistenceMetrics.noteError("Succeeded only on attempt "+(i+1)+": "+lastException);
+                }
+                managementStateReadPersistenceMetrics.noteSuccess(Duration.of(timer));
                 return result;
             } catch (IOException e) {
                 if (i < (maxLoadAttempts - 1)) {
@@ -889,7 +944,11 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
                 lastException = e;
             }
         }
-        throw new IllegalStateException("Failed to load mangement-plane memento "+maxLoadAttempts+" consecutive times", lastException);
+        String message = "Failed to load mangement-plane memento "+maxLoadAttempts+" consecutive times";
+        managementStateReadPersistenceMetrics.noteError(message+": "+lastException);
+        managementStateReadPersistenceMetrics.noteFailure(Duration.of(timer));
+
+        throw new IllegalStateException(message, lastException);
     }
 
     protected ManagementNodeSyncRecord createManagementNodeSyncRecord(boolean useLocalTimestampAsRemoteTimestamp) {
@@ -944,4 +1003,26 @@ public class HighAvailabilityManagerImpl implements HighAvailabilityManager {
     public String toString() {
         return super.toString()+"[node:"+ownNodeId+";running="+running+"]";
     }
+    
+    @Override
+    public Map<String,Object> getMetrics() {
+        Map<String,Object> result = MutableMap.of();
+        
+        result.put("state", getNodeState());
+        result.put("uptime", Time.makeTimeStringRounded(Duration.millis(currentTimeMillis()-startTimeUtc)));
+        result.put("currentTimeUtc", currentTimeMillis());
+        result.put("startTimeUtc", startTimeUtc);
+        result.put("highAvailability", MutableMap.<String,Object>of(
+            "priority", getPriority(),
+            "pollPeriod", getPollPeriod().toMilliseconds(),
+            "heartbeatTimeout", getHeartbeatTimeout().toMilliseconds(),
+            "history", nodeStateHistory));
+        
+        result.putAll(managementContext.getRebindManager().getMetrics());
+        result.put("managementStatePersistence", 
+            MutableMap.of("read", managementStateReadPersistenceMetrics, "write", managementStateWritePersistenceMetrics));
+        
+        return result;
+    }
+    
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java b/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
index 2771475..c04a31c 100644
--- a/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
+++ b/core/src/main/java/brooklyn/management/internal/NonDeploymentManagementContext.java
@@ -530,6 +530,11 @@ public class NonDeploymentManagementContext implements ManagementContextInternal
         public BrooklynMementoRawData retrieveMementoRawData() {
             throw new IllegalStateException("Non-deployment context "+NonDeploymentManagementContext.this+" is not valid for this operation.");
         }
+
+        @Override
+        public Map<String, Object> getMetrics() {
+            throw new IllegalStateException("Non-deployment context "+NonDeploymentManagementContext.this+" is not valid for this operation.");
+        }
     }
 
     /**
@@ -590,6 +595,10 @@ public class NonDeploymentManagementContext implements ManagementContextInternal
         public long getPriority() {
             throw new IllegalStateException("Non-deployment context "+NonDeploymentManagementContext.this+" is not valid for this operation.");
         }
+        @Override
+        public Map<String, Object> getMetrics() {
+            throw new IllegalStateException("Non-deployment context "+NonDeploymentManagementContext.this+" is not valid for this operation.");
+        }
     }
     
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/usage/rest-api/src/main/java/brooklyn/rest/api/ServerApi.java
----------------------------------------------------------------------
diff --git a/usage/rest-api/src/main/java/brooklyn/rest/api/ServerApi.java b/usage/rest-api/src/main/java/brooklyn/rest/api/ServerApi.java
index 7977ea8..246e31a 100644
--- a/usage/rest-api/src/main/java/brooklyn/rest/api/ServerApi.java
+++ b/usage/rest-api/src/main/java/brooklyn/rest/api/ServerApi.java
@@ -18,6 +18,8 @@
  */
 package brooklyn.rest.api;
 
+import java.util.Map;
+
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.FormParam;
@@ -97,6 +99,11 @@ public interface ServerApi {
     @ApiOperation(value = "Returns the HA state of this management node")
     public ManagementNodeState getHighAvailabilityNodeState();
     
+    @GET
+    @Path("/ha/metrics")
+    @ApiOperation(value = "Returns a collection of HA metrics")
+    public Map<String,Object> getHighAvailabilityMetrics();
+    
     @POST
     @Path("/ha/state")
     @ApiOperation(value = "Changes the HA state of this management node")

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b9c1b6fc/usage/rest-server/src/main/java/brooklyn/rest/resources/ServerResource.java
----------------------------------------------------------------------
diff --git a/usage/rest-server/src/main/java/brooklyn/rest/resources/ServerResource.java b/usage/rest-server/src/main/java/brooklyn/rest/resources/ServerResource.java
index 87fce42..50f99e1 100644
--- a/usage/rest-server/src/main/java/brooklyn/rest/resources/ServerResource.java
+++ b/usage/rest-server/src/main/java/brooklyn/rest/resources/ServerResource.java
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -57,6 +58,7 @@ import brooklyn.rest.domain.VersionSummary;
 import brooklyn.rest.transform.HighAvailabilityTransformer;
 import brooklyn.rest.util.WebResourceUtils;
 import brooklyn.util.ResourceUtils;
+import brooklyn.util.collections.MutableMap;
 import brooklyn.util.exceptions.Exceptions;
 import brooklyn.util.file.ArchiveBuilder;
 import brooklyn.util.flags.TypeCoercions;
@@ -255,6 +257,11 @@ public class ServerResource extends AbstractBrooklynRestResource implements Serv
     }
 
     @Override
+    public Map<String, Object> getHighAvailabilityMetrics() {
+        return mgmt().getHighAvailabilityManager().getMetrics();
+    }
+    
+    @Override
     public long getHighAvailabitlityPriority() {
         return mgmt().getHighAvailabilityManager().getPriority();
     }


Mime
View raw message