brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From henev...@apache.org
Subject [18/26] git commit: rewrite of ServiceFailureDetector -- part of what it did before is now done by ServiceStateLogic of course, so now this extends that, and provides options for emitting notifications of ENTITY_FAILED, ENTITY_RECOVERED, as well as suppr
Date Fri, 29 Aug 2014 23:01:18 GMT
rewrite of ServiceFailureDetector -- part of what it did before is now done by ServiceStateLogic of course,
so now this extends that, and provides options for emitting notifications of ENTITY_FAILED, ENTITY_RECOVERED, as well as suppressing ON_FIRE


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/b2daedf8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/b2daedf8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/b2daedf8

Branch: refs/heads/master
Commit: b2daedf8336a891167602f67e7f8e576256dc5e8
Parents: d3886a0
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Mon Aug 25 15:17:53 2014 -0500
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Wed Aug 27 02:17:18 2014 -0400

----------------------------------------------------------------------
 api/src/main/java/brooklyn/entity/Entity.java   |   2 +
 .../entity/basic/ServiceStateLogic.java         |  33 +-
 .../internal/LocalSubscriptionManager.java      |   6 +-
 .../util/task/BasicExecutionManager.java        |   7 +-
 .../enricher/CustomAggregatingEnricherTest.java |   2 +-
 .../basic/MultiLocationResolverTest.java        |   3 +-
 .../brooklyn/test/entity/TestClusterImpl.java   |   9 +
 .../java/brooklyn/test/entity/TestEntity.java   |   4 +-
 .../brooklyn/test/entity/TestEntityImpl.java    |  10 +-
 .../brooklyn/demo/CumulusRDFApplication.java    |   6 +-
 .../demo/HighAvailabilityCassandraCluster.java  |   5 +-
 .../java/brooklyn/demo/ResilientMongoDbApp.java |   2 +-
 .../java/brooklyn/demo/RiakClusterExample.java  |   9 +-
 .../brooklyn/demo/WideAreaCassandraCluster.java |   5 +-
 .../policy/ha/ServiceFailureDetector.java       | 409 +++++--------------
 .../entity/brooklyn/BrooklynMetricsTest.java    |  20 +-
 .../autoscaling/AutoScalerPolicyMetricTest.java |   5 +-
 .../brooklyn/policy/ha/HaPolicyRebindTest.java  |   9 +-
 ...ServiceFailureDetectorStabilizationTest.java |  31 +-
 .../policy/ha/ServiceFailureDetectorTest.java   | 203 +++++----
 .../brooklyn/policy/ha/ServiceReplacerTest.java |   2 +-
 .../brooklyn/entity/webapp/JBossExample.groovy  |  48 ---
 .../test/entity/TestJavaWebAppEntity.groovy     |  75 ----
 .../test/entity/TestJavaWebAppEntity.java       |  73 ++++
 24 files changed, 421 insertions(+), 557 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/api/src/main/java/brooklyn/entity/Entity.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/brooklyn/entity/Entity.java b/api/src/main/java/brooklyn/entity/Entity.java
index 88a10fa..9a09392 100644
--- a/api/src/main/java/brooklyn/entity/Entity.java
+++ b/api/src/main/java/brooklyn/entity/Entity.java
@@ -25,6 +25,8 @@ import java.util.Set;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import com.google.common.annotations.Beta;
+
 import brooklyn.basic.BrooklynObject;
 import brooklyn.config.ConfigKey;
 import brooklyn.config.ConfigKey.HasConfigKey;

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java b/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java
index 9e66381..e3d5090 100644
--- a/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java
+++ b/core/src/main/java/brooklyn/entity/basic/ServiceStateLogic.java
@@ -70,6 +70,12 @@ public class ServiceStateLogic {
     /** static only; not for instantiation */
     private ServiceStateLogic() {}
 
+    public static <TKey,TVal> TVal getMapSensorEntry(EntityLocal entity, AttributeSensor<Map<TKey,TVal>> sensor, TKey key) {
+        Map<TKey, TVal> map = entity.getAttribute(sensor);
+        if (map==null) return null;
+        return map.get(key);
+    }
+    
     @SuppressWarnings("unchecked")
     public static <TKey,TVal> void clearMapSensorEntry(EntityLocal entity, AttributeSensor<Map<TKey,TVal>> sensor, TKey key) {
         updateMapSensorEntry(entity, sensor, key, (TVal)Entities.REMOVE);
@@ -164,6 +170,15 @@ public class ServiceStateLogic {
      * {@link ServiceStateLogic#newEnricherForServiceState(Class)} and added to an entity.
      */
     public static class ComputeServiceState extends AbstractEnricher implements SensorEventListener<Object> {
+        public ComputeServiceState() {}
+        public ComputeServiceState(Map<?,?> flags) { super(flags); }
+            
+        @Override
+        public void init() {
+            super.init();
+            if (uniqueTag==null) uniqueTag = "service.state.actual";
+        }
+        
         public void setEntity(EntityLocal entity) {
             super.setEntity(entity);
             if (suppressDuplicates==null) {
@@ -206,7 +221,13 @@ public class ServiceStateLogic {
             } else if (problems!=null && !problems.isEmpty()) {
                 return Lifecycle.ON_FIRE;
             } else {
-                return (up==null ? null : up ? Lifecycle.RUNNING : Lifecycle.STOPPED);
+                // no expected transition
+                // if the problems map is non-null, then infer, else leave unchanged
+                if (problems!=null)
+                    return (up==null ? null /* remove if up is not set */ : 
+                        up ? Lifecycle.RUNNING : Lifecycle.STOPPED);
+                else
+                    return entity.getAttribute(SERVICE_STATE_ACTUAL);
             }
         }
 
@@ -220,7 +241,7 @@ public class ServiceStateLogic {
         return newEnricherForServiceState(ComputeServiceState.class);
     }
     public static final EnricherSpec<?> newEnricherForServiceState(Class<? extends Enricher> type) {
-        return EnricherSpec.create(type).uniqueTag("service.state.actual from service.state.expected and service.problems");
+        return EnricherSpec.create(type);
     }
     
     public static class ServiceProblemsLogic {
@@ -244,6 +265,14 @@ public class ServiceStateLogic {
         public static void clearProblemsIndicator(EntityLocal entity, Effector<?> eff) {
             clearMapSensorEntry(entity, Attributes.SERVICE_PROBLEMS, eff.getName());
         }
+        /** as {@link #updateProblemsIndicator(EntityLocal, Sensor, Object)} */
+        public static void updateProblemsIndicator(EntityLocal entity, String key, Object value) {
+            updateMapSensorEntry(entity, Attributes.SERVICE_PROBLEMS, key, value);
+        }
+        /** as {@link #clearProblemsIndicator(EntityLocal, Sensor)} */
+        public static void clearProblemsIndicator(EntityLocal entity, String key) {
+            clearMapSensorEntry(entity, Attributes.SERVICE_PROBLEMS, key);
+        }
     }
     
     public static class ComputeServiceIndicatorsFromChildrenAndMembers extends AbstractMultipleSensorAggregator<Void> implements SensorEventListener<Object> {

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java b/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java
index 0bd8578..83cec75 100644
--- a/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java
+++ b/core/src/main/java/brooklyn/management/internal/LocalSubscriptionManager.java
@@ -185,7 +185,11 @@ public class LocalSubscriptionManager extends AbstractSubscriptionManager {
                         return "LSM.publish("+event+")";
                     }
                     public void run() {
-                        sAtClosureCreation.listener.onEvent(event);
+                        try {
+                            sAtClosureCreation.listener.onEvent(event);
+                        } catch (Throwable t) {
+                            LOG.warn("Error in "+this+": "+t, t);
+                        }
                     }});
                 totalEventsDeliveredCount.incrementAndGet();
             }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java b/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java
index da8c456..2f6e396 100644
--- a/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java
+++ b/core/src/main/java/brooklyn/util/task/BasicExecutionManager.java
@@ -397,8 +397,13 @@ public class BasicExecutionManager implements ExecutionManager {
                         afterEnd(flags, task);
                     }
                     if (error!=null) {
+                        /* we throw, after logging debug.
+                         * the throw means the error is available for task submitters to monitor.
+                         * however it is possible no one is monitoring it, in which case we will have debug logging only for errors.
+                         * (the alternative, of warn-level logging in lots of places where we don't want it, seems worse!) 
+                         */
                         if (log.isDebugEnabled()) {
-                            // debug only here, because we rethrow
+                            // debug only here, because most submitters will handle failures
                             log.debug("Exception running task "+task+" (rethrowing): "+error.getMessage(), error);
                             if (log.isTraceEnabled())
                                 log.trace("Trace for exception running task "+task+" (rethrowing): "+error.getMessage(), error);

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java b/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java
index f3e06f6..70d0c10 100644
--- a/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java
+++ b/core/src/test/java/brooklyn/enricher/CustomAggregatingEnricherTest.java
@@ -48,7 +48,7 @@ public class CustomAggregatingEnricherTest extends BrooklynAppUnitTestSupport {
     public static final Logger log = LoggerFactory.getLogger(CustomAggregatingEnricherTest.class);
             
     private static final long TIMEOUT_MS = 10*1000;
-    private static final long SHORT_WAIT_MS = 250;
+    private static final long SHORT_WAIT_MS = 50;
     
     TestEntity entity;
     SimulatedLocation loc;

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java b/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java
index 937679d..85bcfbd 100644
--- a/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java
+++ b/core/src/test/java/brooklyn/location/basic/MultiLocationResolverTest.java
@@ -44,6 +44,7 @@ import brooklyn.location.MachineProvisioningLocation;
 import brooklyn.location.NoMachinesAvailableException;
 import brooklyn.location.cloud.AvailabilityZoneExtension;
 import brooklyn.management.internal.LocalManagementContext;
+import brooklyn.test.entity.LocalManagementContextForTests;
 import brooklyn.util.collections.MutableList;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.exceptions.Exceptions;
@@ -63,7 +64,7 @@ public class MultiLocationResolverTest {
 
     @BeforeMethod(alwaysRun=true)
     public void setUp() throws Exception {
-        managementContext = new LocalManagementContext(BrooklynProperties.Factory.newEmpty());
+        managementContext = LocalManagementContextForTests.newInstance();
         brooklynProperties = managementContext.getBrooklynProperties();
     }
     

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java b/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java
index 0bd2521..3663520 100644
--- a/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java
+++ b/core/src/test/java/brooklyn/test/entity/TestClusterImpl.java
@@ -18,6 +18,7 @@
  */
 package brooklyn.test.entity;
 
+import brooklyn.entity.basic.QuorumCheck.QuorumChecks;
 import brooklyn.entity.group.DynamicClusterImpl;
 import brooklyn.entity.trait.Startable;
 
@@ -38,6 +39,14 @@ public class TestClusterImpl extends DynamicClusterImpl implements TestCluster {
     }
     
     @Override
+    protected void initEnrichers() {
+        // say this is up if it has no children 
+        setConfig(UP_QUORUM_CHECK, QuorumChecks.atLeastOneUnlessEmpty());
+        
+        super.initEnrichers();
+    }
+    
+    @Override
     public Integer resize(Integer desiredSize) {
         this.size = desiredSize;
         return size;

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/test/entity/TestEntity.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/test/entity/TestEntity.java b/core/src/test/java/brooklyn/test/entity/TestEntity.java
index ac88acd..c3ca9c2 100644
--- a/core/src/test/java/brooklyn/test/entity/TestEntity.java
+++ b/core/src/test/java/brooklyn/test/entity/TestEntity.java
@@ -76,7 +76,9 @@ public interface TestEntity extends Entity, Startable, EntityLocal, EntityIntern
     public static final AttributeSensor<String> NAME = Sensors.newStringSensor("test.name", "Test name");
     public static final BasicNotificationSensor<Integer> MY_NOTIF = new BasicNotificationSensor<Integer>(Integer.class, "test.myNotif", "Test notification");
     
-    public static final AttributeSensor<Lifecycle> SERVICE_STATE = Attributes.SERVICE_STATE;
+    public static final AttributeSensor<Lifecycle> SERVICE_STATE_ACTUAL = Attributes.SERVICE_STATE_ACTUAL;
+    @Deprecated
+    public static final AttributeSensor<Lifecycle> SERVICE_STATE = Attributes.SERVICE_STATE_ACTUAL;
     
     public static final MethodEffector<Void> MY_EFFECTOR = new MethodEffector<Void>(TestEntity.class, "myEffector");
     public static final MethodEffector<Object> IDENTITY_EFFECTOR = new MethodEffector<Object>(TestEntity.class, "identityEffector");

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java b/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java
index c2a3884..3ccf614 100644
--- a/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java
+++ b/core/src/test/java/brooklyn/test/entity/TestEntityImpl.java
@@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory;
 import brooklyn.entity.Entity;
 import brooklyn.entity.basic.AbstractEntity;
 import brooklyn.entity.basic.Lifecycle;
+import brooklyn.entity.basic.ServiceStateLogic;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.location.Location;
 import brooklyn.util.collections.MutableMap;
@@ -122,19 +123,20 @@ public class TestEntityImpl extends AbstractEntity implements TestEntity {
     public void start(Collection<? extends Location> locs) {
         LOG.trace("Starting {}", this);
         callHistory.add("start");
-        setAttribute(SERVICE_STATE, Lifecycle.STARTING);
+        ServiceStateLogic.setExpectedState(this, Lifecycle.STARTING);
         counter.incrementAndGet();
         addLocations(locs);
-        setAttribute(SERVICE_STATE, Lifecycle.RUNNING);
+        ServiceStateLogic.setExpectedState(this, Lifecycle.RUNNING);
+        setAttribute(SERVICE_UP, true);
     }
 
     @Override
     public void stop() { 
         LOG.trace("Stopping {}", this);
         callHistory.add("stop");
-        setAttribute(SERVICE_STATE, Lifecycle.STOPPING);
+        ServiceStateLogic.setExpectedState(this, Lifecycle.STOPPING);
         counter.decrementAndGet();
-        setAttribute(SERVICE_STATE, Lifecycle.STOPPED);
+        ServiceStateLogic.setExpectedState(this, Lifecycle.STOPPED);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java
----------------------------------------------------------------------
diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java
index a54ef58..eaabaff 100644
--- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java
+++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/CumulusRDFApplication.java
@@ -31,15 +31,12 @@ import brooklyn.config.ConfigKey;
 import brooklyn.entity.Effector;
 import brooklyn.entity.Entity;
 import brooklyn.entity.basic.AbstractApplication;
-import brooklyn.entity.basic.Attributes;
 import brooklyn.entity.basic.ConfigKeys;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.EntityInternal;
 import brooklyn.entity.basic.Lifecycle;
 import brooklyn.entity.basic.ServiceStateLogic;
-import brooklyn.entity.basic.SoftwareProcess;
 import brooklyn.entity.basic.StartableApplication;
-import brooklyn.entity.basic.ServiceStateLogic.ServiceNotUpLogic;
 import brooklyn.entity.effector.EffectorBody;
 import brooklyn.entity.effector.Effectors;
 import brooklyn.entity.java.UsesJava;
@@ -58,6 +55,7 @@ import brooklyn.event.basic.DependentConfiguration;
 import brooklyn.launcher.BrooklynLauncher;
 import brooklyn.location.Location;
 import brooklyn.location.basic.PortRanges;
+import brooklyn.policy.EnricherSpec;
 import brooklyn.policy.PolicySpec;
 import brooklyn.policy.ha.ServiceFailureDetector;
 import brooklyn.policy.ha.ServiceReplacer;
@@ -127,7 +125,7 @@ public class CumulusRDFApplication extends AbstractApplication {
                         .configure(UsesJmx.JMX_PORT, PortRanges.fromString("11099+"))
                         .configure(UsesJmx.RMI_REGISTRY_PORT, PortRanges.fromString("9001+"))
                         .configure(CassandraNode.THRIFT_PORT, PortRanges.fromInteger(getConfig(CASSANDRA_THRIFT_PORT)))
-                        .policy(PolicySpec.create(ServiceFailureDetector.class))
+                        .enricher(EnricherSpec.create(ServiceFailureDetector.class))
                         .policy(PolicySpec.create(ServiceRestarter.class)
                                 .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED)))
                 .policy(PolicySpec.create(ServiceReplacer.class)

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java
----------------------------------------------------------------------
diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java
index 2dce1ea..c21c4cf 100644
--- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java
+++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/HighAvailabilityCassandraCluster.java
@@ -23,14 +23,15 @@ import java.util.List;
 import brooklyn.catalog.Catalog;
 import brooklyn.catalog.CatalogConfig;
 import brooklyn.config.ConfigKey;
-import brooklyn.entity.basic.ConfigKeys;
 import brooklyn.entity.basic.AbstractApplication;
+import brooklyn.entity.basic.ConfigKeys;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.StartableApplication;
 import brooklyn.entity.nosql.cassandra.CassandraDatacenter;
 import brooklyn.entity.nosql.cassandra.CassandraNode;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.launcher.BrooklynLauncher;
+import brooklyn.policy.EnricherSpec;
 import brooklyn.policy.PolicySpec;
 import brooklyn.policy.ha.ServiceFailureDetector;
 import brooklyn.policy.ha.ServiceReplacer;
@@ -64,7 +65,7 @@ public class HighAvailabilityCassandraCluster extends AbstractApplication {
                 //.configure(CassandraCluster.AVAILABILITY_ZONE_NAMES, ImmutableList.of("us-east-1b", "us-east-1c", "us-east-1e"))
                 .configure(CassandraDatacenter.ENDPOINT_SNITCH_NAME, "GossipingPropertyFileSnitch")
                 .configure(CassandraDatacenter.MEMBER_SPEC, EntitySpec.create(CassandraNode.class)
-                        .policy(PolicySpec.create(ServiceFailureDetector.class))
+                        .enricher(EnricherSpec.create(ServiceFailureDetector.class))
                         .policy(PolicySpec.create(ServiceRestarter.class)
                                 .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED)))
                 .policy(PolicySpec.create(ServiceReplacer.class)

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java
----------------------------------------------------------------------
diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java
index 8506b27..290f25e 100644
--- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java
+++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/ResilientMongoDbApp.java
@@ -84,7 +84,7 @@ public class ResilientMongoDbApp extends AbstractApplication implements Startabl
 
     /** invoked whenever a new MongoDB server is added (the server may not be started yet) */
     protected void initSoftwareProcess(SoftwareProcess p) {
-        p.addPolicy(new ServiceFailureDetector());
+        p.addEnricher(new ServiceFailureDetector());
         p.addPolicy(new ServiceRestarter(ServiceFailureDetector.ENTITY_FAILED));
     }
     

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java
----------------------------------------------------------------------
diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java
index d53ce86..0134e27 100644
--- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java
+++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/RiakClusterExample.java
@@ -20,9 +20,6 @@ package brooklyn.demo;
 
 import java.util.List;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
 import brooklyn.catalog.Catalog;
 import brooklyn.catalog.CatalogConfig;
 import brooklyn.config.ConfigKey;
@@ -34,11 +31,15 @@ import brooklyn.entity.nosql.riak.RiakCluster;
 import brooklyn.entity.nosql.riak.RiakNode;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.launcher.BrooklynLauncher;
+import brooklyn.policy.EnricherSpec;
 import brooklyn.policy.PolicySpec;
 import brooklyn.policy.ha.ServiceFailureDetector;
 import brooklyn.policy.ha.ServiceRestarter;
 import brooklyn.util.CommandLineUtil;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
 @Catalog(name = "Riak Cluster Application", description = "Riak ring deployment blueprint")
 public class RiakClusterExample extends AbstractApplication {
 
@@ -67,7 +68,7 @@ public class RiakClusterExample extends AbstractApplication {
         addChild(EntitySpec.create(RiakCluster.class)
                 .configure(RiakCluster.INITIAL_SIZE, getConfig(RIAK_RING_SIZE))
                 .configure(RiakCluster.MEMBER_SPEC, EntitySpec.create(RiakNode.class)
-                        .policy(PolicySpec.create(ServiceFailureDetector.class))
+                        .enricher(EnricherSpec.create(ServiceFailureDetector.class))
                         .policy(PolicySpec.create(ServiceRestarter.class)
                                 .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED))));
     }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java
----------------------------------------------------------------------
diff --git a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java
index c40eb27..8e30fc4 100644
--- a/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java
+++ b/examples/simple-nosql-cluster/src/main/java/brooklyn/demo/WideAreaCassandraCluster.java
@@ -24,8 +24,8 @@ import java.util.List;
 import brooklyn.catalog.Catalog;
 import brooklyn.catalog.CatalogConfig;
 import brooklyn.config.ConfigKey;
-import brooklyn.entity.basic.ConfigKeys;
 import brooklyn.entity.basic.AbstractApplication;
+import brooklyn.entity.basic.ConfigKeys;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.StartableApplication;
 import brooklyn.entity.nosql.cassandra.CassandraDatacenter;
@@ -33,6 +33,7 @@ import brooklyn.entity.nosql.cassandra.CassandraFabric;
 import brooklyn.entity.nosql.cassandra.CassandraNode;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.launcher.BrooklynLauncher;
+import brooklyn.policy.EnricherSpec;
 import brooklyn.policy.PolicySpec;
 import brooklyn.policy.ha.ServiceFailureDetector;
 import brooklyn.policy.ha.ServiceReplacer;
@@ -61,7 +62,7 @@ public class WideAreaCassandraCluster extends AbstractApplication {
                 .configure(CassandraNode.CUSTOM_SNITCH_JAR_URL, "classpath://brooklyn/entity/nosql/cassandra/cassandra-multicloud-snitch.jar")
                 .configure(CassandraFabric.MEMBER_SPEC, EntitySpec.create(CassandraDatacenter.class)
                         .configure(CassandraDatacenter.MEMBER_SPEC, EntitySpec.create(CassandraNode.class)
-                                .policy(PolicySpec.create(ServiceFailureDetector.class))
+                                .enricher(EnricherSpec.create(ServiceFailureDetector.class))
                                 .policy(PolicySpec.create(ServiceRestarter.class)
                                         .configure(ServiceRestarter.FAILURE_SENSOR_TO_MONITOR, ServiceFailureDetector.ENTITY_FAILED)))
                         .policy(PolicySpec.create(ServiceReplacer.class)

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java
----------------------------------------------------------------------
diff --git a/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java b/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java
index 5efad2b..c523aba 100644
--- a/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java
+++ b/policy/src/main/java/brooklyn/policy/ha/ServiceFailureDetector.java
@@ -18,13 +18,10 @@
  */
 package brooklyn.policy.ha;
 
-import static brooklyn.util.time.Time.makeTimeStringRounded;
-
-import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,15 +30,12 @@ import brooklyn.config.ConfigKey;
 import brooklyn.entity.basic.Attributes;
 import brooklyn.entity.basic.ConfigKeys;
 import brooklyn.entity.basic.EntityInternal;
-import brooklyn.entity.basic.EntityLocal;
 import brooklyn.entity.basic.Lifecycle;
-import brooklyn.entity.trait.Startable;
+import brooklyn.entity.basic.ServiceStateLogic;
+import brooklyn.entity.basic.ServiceStateLogic.ComputeServiceState;
 import brooklyn.event.SensorEvent;
-import brooklyn.event.SensorEventListener;
 import brooklyn.event.basic.BasicConfigKey;
 import brooklyn.event.basic.BasicNotificationSensor;
-import brooklyn.management.SubscriptionHandle;
-import brooklyn.policy.basic.AbstractPolicy;
 import brooklyn.policy.ha.HASensors.FailureDescriptor;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.config.ConfigBag;
@@ -50,26 +44,17 @@ import brooklyn.util.flags.SetFromFlag;
 import brooklyn.util.task.BasicTask;
 import brooklyn.util.task.ScheduledTask;
 import brooklyn.util.time.Duration;
-import brooklyn.util.time.Time;
-
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
 
-/** attaches to a SoftwareProcess (or anything emitting SERVICE_UP and SERVICE_STATE)
- * and emits HASensors.ENTITY_FAILED and ENTITY_RECOVERED as appropriate
- * @see MemberFailureDetectionPolicy
+/** 
+ * emits {@link HASensors#ENTITY_FAILED} whenever the parent's default logic ({@link ComputeServiceState}) would detect a problem,
+ * and similarly {@link HASensors#ENTITY_RECOVERED} when recovered.
+ * <p>
+ * gives more control over suppressing {@link Lifecycle#ON_FIRE}, 
+ * for some period of time
+ * (or until another process manually sets {@link Attributes#SERVICE_STATE_ACTUAL} to {@value Lifecycle#ON_FIRE},
+ * which this enricher will not clear until all problems have gone away)
  */
-public class ServiceFailureDetector extends AbstractPolicy {
-
-    // TODO Remove duplication between this and MemberFailureDetectionPolicy.
-    // The latter could be re-written to use this. Or could even be deprecated
-    // in favour of this.
-
-    public enum LastPublished {
-        NONE,
-        FAILED,
-        RECOVERED;
-    }
+public class ServiceFailureDetector extends ServiceStateLogic.ComputeServiceState {
 
     private static final Logger LOG = LoggerFactory.getLogger(ServiceFailureDetector.class);
 
@@ -77,47 +62,45 @@ public class ServiceFailureDetector extends AbstractPolicy {
 
     public static final BasicNotificationSensor<FailureDescriptor> ENTITY_FAILED = HASensors.ENTITY_FAILED;
 
-    // TODO delay before reporting failure (give it time to fix itself, e.g. transient failures)
-    
     @SetFromFlag("onlyReportIfPreviouslyUp")
-    public static final ConfigKey<Boolean> ONLY_REPORT_IF_PREVIOUSLY_UP = ConfigKeys.newBooleanConfigKey("onlyReportIfPreviouslyUp", "", true);
+    public static final ConfigKey<Boolean> ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP = ConfigKeys.newBooleanConfigKey("onlyReportIfPreviouslyUp", 
+        "Prevents the policy from emitting ENTITY_FAILED if the entity fails on startup (ie has never been up)", true);
     
-    @SetFromFlag("useServiceStateRunning")
-    public static final ConfigKey<Boolean> USE_SERVICE_STATE_RUNNING = ConfigKeys.newBooleanConfigKey("useServiceStateRunning", "", true);
+    public static final ConfigKey<Boolean> MONITOR_SERVICE_PROBLEMS = ConfigKeys.newBooleanConfigKey("monitorServiceProblems", 
+        "Whether to monitor service problems, and emit on failures there (if set to false, this monitors only service up)", true);
 
-    @SetFromFlag("setOnFireOnFailure")
-    public static final ConfigKey<Boolean> SET_ON_FIRE_ON_FAILURE = ConfigKeys.newBooleanConfigKey("setOnFireOnFailure", "", true);
+    @SetFromFlag("serviceOnFireStabilizationDelay")
+    public static final ConfigKey<Duration> SERVICE_ON_FIRE_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class)
+            .name("serviceOnFire.stabilizationDelay")
+            .description("Time period for which the service must be consistently down for (e.g. doesn't report down-up-down) before concluding ON_FIRE")
+            .defaultValue(Duration.ZERO)
+            .build();
 
-    @SetFromFlag("serviceFailedStabilizationDelay")
-    public static final ConfigKey<Duration> SERVICE_FAILED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class)
-            .name("serviceRestarter.serviceFailedStabilizationDelay")
-            .description("Time period for which the service must be consistently down for (e.g. doesn't report down-up-down) before concluding failure")
+    @SetFromFlag("entityFailedStabilizationDelay")
+    public static final ConfigKey<Duration> ENTITY_FAILED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class)
+            .name("entityFailed.stabilizationDelay")
+            .description("Time period for which the service must be consistently down for (e.g. doesn't report down-up-down) before emitting ENTITY_FAILED")
             .defaultValue(Duration.ZERO)
             .build();
 
-    @SetFromFlag("serviceRecoveredStabilizationDelay")
-    public static final ConfigKey<Duration> SERVICE_RECOVERED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class)
-            .name("serviceRestarter.serviceRecoveredStabilizationDelay")
-            .description("For a failed entity, time period for which the service must be consistently up for (e.g. doesn't report up-down-up) before concluding recovered")
+    @SetFromFlag("entityRecoveredStabilizationDelay")
+    public static final ConfigKey<Duration> ENTITY_RECOVERED_STABILIZATION_DELAY = BasicConfigKey.builder(Duration.class)
+            .name("entityRecovered.stabilizationDelay")
+            .description("For a failed entity, time period for which the service must be consistently up for (e.g. doesn't report up-down-up) before emitting ENTITY_RECOVERED")
             .defaultValue(Duration.ZERO)
             .build();
 
-    protected final AtomicReference<Boolean> serviceIsUp = new AtomicReference<Boolean>();
-    protected final AtomicReference<Lifecycle> serviceState = new AtomicReference<Lifecycle>();
-    protected final AtomicReference<Long> serviceLastUp = new AtomicReference<Long>();
-    protected final AtomicReference<Long> serviceLastDown = new AtomicReference<Long>();
+    protected Long firstUpTime;
     
     protected Long currentFailureStartTime = null;
     protected Long currentRecoveryStartTime = null;
-
-    protected LastPublished lastPublished = LastPublished.NONE;
-    protected boolean weSetItOnFire = false;
+    
+    protected Long publishEntityFailedTime = null;
+    protected Long publishEntityRecoveredTime = null;
 
     private final AtomicBoolean executorQueued = new AtomicBoolean(false);
     private volatile long executorTime = 0;
 
-    private List<SubscriptionHandle> subscriptionHandles = Lists.newCopyOnWriteArrayList();
-
     public ServiceFailureDetector() {
         this(new ConfigBag());
     }
@@ -130,161 +113,98 @@ public class ServiceFailureDetector extends AbstractPolicy {
         // TODO hierarchy should use ConfigBag, and not change flags
         super(configBag.getAllConfigMutable());
     }
-
-    @Override
-    public void setEntity(EntityLocal entity) {
-        super.setEntity(entity);
-        doSubscribe();
-        onMemberAdded();
-    }
-
+    
     @Override
-    public void suspend() {
-        super.suspend();
-        doUnsubscribe();
+    public void onEvent(SensorEvent<Object> event) {
+        if (firstUpTime==null && event!=null && Attributes.SERVICE_UP.equals(event.getSensor()) && Boolean.TRUE.equals(event.getValue())) {
+            firstUpTime = event.getTimestamp();
+        }
+        
+        super.onEvent(event);
     }
     
     @Override
-    public void resume() {
-        serviceIsUp.set(null);
-        serviceState.set(null);
-        serviceLastUp.set(null);
-        serviceLastDown.set(null);
-        currentFailureStartTime = null;
-        currentRecoveryStartTime = null;
-        lastPublished = LastPublished.NONE;
-        weSetItOnFire = false;
-        executorQueued.set(false);
-        executorTime = 0;
-
-        super.resume();
-        doSubscribe();
-        onMemberAdded();
-    }
-
-    protected void doSubscribe() {
-        if (subscriptionHandles.isEmpty()) {
-            if (getConfig(USE_SERVICE_STATE_RUNNING)) {
-                SubscriptionHandle handle = subscribe(entity, Attributes.SERVICE_STATE, new SensorEventListener<Lifecycle>() {
-                    @Override public void onEvent(SensorEvent<Lifecycle> event) {
-                        onServiceState(event.getValue());
-                    }
-                });
-                subscriptionHandles.add(handle);
+    protected void setActualState(Lifecycle state) {
+        if (state==Lifecycle.ON_FIRE) {
+            if (currentFailureStartTime==null) {
+                currentFailureStartTime = System.currentTimeMillis();
+                publishEntityFailedTime = currentFailureStartTime + getConfig(ENTITY_FAILED_STABILIZATION_DELAY).toMilliseconds();
             }
+            // cancel any existing recovery
+            currentRecoveryStartTime = null;
+            publishEntityRecoveredTime = null;
             
-            SubscriptionHandle handle = subscribe(entity, Startable.SERVICE_UP, new SensorEventListener<Boolean>() {
-                @Override public void onEvent(SensorEvent<Boolean> event) {
-                    onServiceUp(event.getValue());
-                }
-            });
-            subscriptionHandles.add(handle);
-        }
-    }
-    
-    protected void doUnsubscribe() {
-        // TODO Could be more defensive with synchronization, but things shouldn't be calling resume + suspend concurrently
-        for (SubscriptionHandle handle : subscriptionHandles) {
-            unsubscribe(entity, handle);
-        }
-        subscriptionHandles.clear();
-    }
-    
-    private Duration getServiceFailedStabilizationDelay() {
-        return getConfig(SERVICE_FAILED_STABILIZATION_DELAY);
-    }
-
-    private Duration getServiceRecoveredStabilizationDelay() {
-        return getConfig(SERVICE_RECOVERED_STABILIZATION_DELAY);
-    }
-
-    private synchronized void onServiceUp(Boolean isNowUp) {
-        if (isNowUp != null) {
-            Boolean old = serviceIsUp.getAndSet(isNowUp);
-            if (isNowUp) {
-                serviceLastUp.set(System.currentTimeMillis());
+            long now = System.currentTimeMillis();
+            
+            long delayBeforeCheck = currentFailureStartTime+getConfig(SERVICE_ON_FIRE_STABILIZATION_DELAY).toMilliseconds() - now;
+            if (delayBeforeCheck<=0) {
+                super.setActualState(state);
             } else {
-                serviceLastDown.set(System.currentTimeMillis());
-            }
-            if (!Objects.equal(old, serviceIsUp)) {
-                checkHealth();
+                recomputeAfterDelay(delayBeforeCheck);
             }
-        }
-    }
-    
-    private synchronized void onServiceState(Lifecycle status) {
-        if (status != null) {
-            Lifecycle old = serviceState.getAndSet(status);
-            if (!Objects.equal(old, status)) {
-                checkHealth();
-            }
-        }
-    }
-    
-    private synchronized void onMemberAdded() {
-        if (getConfig(USE_SERVICE_STATE_RUNNING)) {
-            Lifecycle status = entity.getAttribute(Attributes.SERVICE_STATE);
-            onServiceState(status);
-        }
-        
-        Boolean isUp = entity.getAttribute(Startable.SERVICE_UP);
-        onServiceUp(isUp);
-    }
-
-    private synchronized void checkHealth() {
-        CalculatedStatus status = calculateStatus();
-        boolean failed = status.failed;
-        boolean healthy = status.healthy;
-        long now = System.currentTimeMillis();
-        
-        if (healthy) {
-            if (lastPublished == LastPublished.FAILED) {
-                if (currentRecoveryStartTime == null) {
-                    LOG.info("{} health-check for {}, component now recovering: {}", new Object[] {this, entity, status.getDescription()});
-                    currentRecoveryStartTime = now;
-                    schedulePublish();
+            
+            if (publishEntityFailedTime!=null) {
+                delayBeforeCheck = publishEntityFailedTime - now;
+                if (firstUpTime==null && getConfig(ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP)) {
+                    // suppress
+                    publishEntityFailedTime = null;
+                } else if (delayBeforeCheck<=0) {
+                    publishEntityFailedTime = null;
+                    entity.emit(HASensors.ENTITY_FAILED, new HASensors.FailureDescriptor(entity, getFailureDescription(now)));
                 } else {
-                    if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component continuing recovering: {}", new Object[] {this, entity, status.getDescription()});
+                    recomputeAfterDelay(delayBeforeCheck);
                 }
-            } else {
-                if (currentFailureStartTime != null) {
-                    LOG.info("{} health-check for {}, component now healthy: {}", new Object[] {this, entity, status.getDescription()});
+            }
+            
+        } else {
+            if (state == Lifecycle.RUNNING) {
+                if (currentFailureStartTime!=null) {
                     currentFailureStartTime = null;
-                } else {
-                    if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component still healthy: {}", new Object[] {this, entity, status.getDescription()});
+                    publishEntityFailedTime = null;
+
+                    currentRecoveryStartTime = System.currentTimeMillis();
+                    publishEntityRecoveredTime = currentRecoveryStartTime + getConfig(ENTITY_RECOVERED_STABILIZATION_DELAY).toMilliseconds();
                 }
             }
-        } else if (failed) {
-            if (lastPublished != LastPublished.FAILED) {
-                if (currentFailureStartTime == null) {
-                    LOG.info("{} health-check for {}, component now failing: {}", new Object[] {this, entity, status.getDescription()});
-                    currentFailureStartTime = now;
-                    schedulePublish();
-                } else {
-                    if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component continuing failing: {}", new Object[] {this, entity, status.getDescription()});
-                }
-            } else {
-                if (currentRecoveryStartTime != null) {
-                    LOG.info("{} health-check for {}, component now failing: {}", new Object[] {this, entity, status.getDescription()});
-                    currentRecoveryStartTime = null;
+
+            super.setActualState(state);
+            
+            if (publishEntityRecoveredTime!=null) {
+                long now = System.currentTimeMillis();
+                long delayBeforeCheck = publishEntityRecoveredTime - now;
+                if (delayBeforeCheck<=0) {
+                    entity.emit(HASensors.ENTITY_RECOVERED, new HASensors.FailureDescriptor(entity, null));
+                    publishEntityRecoveredTime = null;
                 } else {
-                    if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, component still failed: {}", new Object[] {this, entity, status.getDescription()});
+                    recomputeAfterDelay(delayBeforeCheck);
                 }
             }
-        } else {
-            if (LOG.isTraceEnabled()) LOG.trace("{} health-check for {}, in unconfirmed sate: {}", new Object[] {this, entity, status.getDescription()});
         }
     }
-    
-    protected CalculatedStatus calculateStatus() {
-        return new CalculatedStatus();
-    }
 
-    protected void schedulePublish() {
-        schedulePublish(0);
+    private String getFailureDescription(long now) {
+        String description = null;
+        Map<String, Object> serviceProblems = entity.getAttribute(Attributes.SERVICE_PROBLEMS);
+        if (serviceProblems!=null && !serviceProblems.isEmpty()) {
+            Entry<String, Object> problem = serviceProblems.entrySet().iterator().next();
+            description = problem.getKey()+": "+problem.getValue();
+            if (serviceProblems.size()>1) {
+                description = serviceProblems.size()+" service problems, including "+description;
+            } else {
+                description = "service problem: "+description;
+            }
+        } else if (Boolean.FALSE.equals(entity.getAttribute(Attributes.SERVICE_UP))) {
+            description = "service not up";
+        } else {
+            description = "service failure detected";
+        }
+        if (publishEntityFailedTime!=null && currentFailureStartTime!=null && publishEntityFailedTime > currentFailureStartTime)
+            description = " (stabilized for "+Duration.of(now - currentFailureStartTime, TimeUnit.MILLISECONDS)+")";
+        return description;
     }
     
-    protected void schedulePublish(long delay) {
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    protected void recomputeAfterDelay(long delay) {
         if (isRunning() && executorQueued.compareAndSet(false, true)) {
             long now = System.currentTimeMillis();
             delay = Math.max(0, Math.max(delay, (executorTime + MIN_PERIOD_BETWEEN_EXECS_MILLIS) - now));
@@ -296,16 +216,16 @@ public class ServiceFailureDetector extends AbstractPolicy {
                         executorTime = System.currentTimeMillis();
                         executorQueued.set(false);
 
-                        publishNow();
+                        onEvent(null);
                         
                     } catch (Exception e) {
                         if (isRunning()) {
-                            LOG.error("Error resizing: "+e, e);
+                            LOG.error("Error in enricher "+this+": "+e, e);
                         } else {
-                            if (LOG.isDebugEnabled()) LOG.debug("Error resizing, but no longer running: "+e, e);
+                            if (LOG.isDebugEnabled()) LOG.debug("Error in enricher "+this+" (but no longer running): "+e, e);
                         }
                     } catch (Throwable t) {
-                        LOG.error("Error in service-failure-detector: "+t, t);
+                        LOG.error("Error in enricher "+this+": "+t, t);
                         throw Exceptions.propagate(t);
                     }
                 }
@@ -316,115 +236,4 @@ public class ServiceFailureDetector extends AbstractPolicy {
         }
     }
     
-    private synchronized void publishNow() {
-        if (!isRunning()) return;
-        
-        CalculatedStatus calculatedStatus = calculateStatus();
-        
-        Long lastUpTime = serviceLastUp.get();
-        Long lastDownTime = serviceLastDown.get();
-        Boolean isUp = serviceIsUp.get();
-        Lifecycle status = serviceState.get();
-        boolean failed = calculatedStatus.failed;
-        boolean healthy = calculatedStatus.healthy;
-        long serviceFailedStabilizationDelay = getServiceFailedStabilizationDelay().toMilliseconds();
-        long serviceRecoveredStabilizationDelay = getServiceRecoveredStabilizationDelay().toMilliseconds();
-        long now = System.currentTimeMillis();
-        
-        if (failed) {
-            if (lastPublished != LastPublished.FAILED) {
-                // only publish if consistently down for serviceFailedStabilizationDelay
-                long currentFailurePeriod = getTimeDiff(now, currentFailureStartTime);
-                long sinceLastUpPeriod = getTimeDiff(now, lastUpTime);
-                if (currentFailurePeriod > serviceFailedStabilizationDelay && sinceLastUpPeriod > serviceFailedStabilizationDelay) {
-                    String description = calculatedStatus.getDescription();
-                    LOG.warn("{} health-check for {}, publishing component failed: {}", new Object[] {this, entity, description});
-                    if (getConfig(USE_SERVICE_STATE_RUNNING) && getConfig(SET_ON_FIRE_ON_FAILURE) && status != Lifecycle.ON_FIRE) {
-                        weSetItOnFire = true;
-                        entity.setAttribute(Attributes.SERVICE_STATE, Lifecycle.ON_FIRE);
-                    }
-                    entity.emit(HASensors.ENTITY_FAILED, new HASensors.FailureDescriptor(entity, description));
-                    lastPublished = LastPublished.FAILED;
-                    currentRecoveryStartTime = null;
-                } else {
-                    long nextAttemptTime = Math.max(serviceFailedStabilizationDelay - currentFailurePeriod, serviceFailedStabilizationDelay - sinceLastUpPeriod);
-                    schedulePublish(nextAttemptTime);
-                }
-            }
-        } else if (healthy) {
-            if (lastPublished == LastPublished.FAILED) {
-                // only publish if consistently up for serviceRecoveredStabilizationDelay
-                long currentRecoveryPeriod = getTimeDiff(now, currentRecoveryStartTime);
-                long sinceLastDownPeriod = getTimeDiff(now, lastDownTime);
-                if (currentRecoveryPeriod > serviceRecoveredStabilizationDelay && sinceLastDownPeriod > serviceRecoveredStabilizationDelay) {
-                    String description = calculatedStatus.getDescription();
-                    LOG.warn("{} health-check for {}, publishing component recovered: {}", new Object[] {this, entity, description});
-                    if (weSetItOnFire) {
-                        if (status == Lifecycle.ON_FIRE) {
-                            entity.setAttribute(Attributes.SERVICE_STATE, Lifecycle.RUNNING);
-                        }
-                        weSetItOnFire = false;
-                    }
-                    entity.emit(HASensors.ENTITY_RECOVERED, new HASensors.FailureDescriptor(entity, description));
-                    lastPublished = LastPublished.RECOVERED;
-                    currentFailureStartTime = null;
-                } else {
-                    long nextAttemptTime = Math.max(serviceRecoveredStabilizationDelay - currentRecoveryPeriod, serviceRecoveredStabilizationDelay - sinceLastDownPeriod);
-                    schedulePublish(nextAttemptTime);
-                }
-            }
-        }
-    }
-
-    public class CalculatedStatus {
-        public final boolean failed;
-        public final boolean healthy;
-        
-        public CalculatedStatus() {
-            Long lastUpTime = serviceLastUp.get();
-            Boolean isUp = serviceIsUp.get();
-            Lifecycle status = serviceState.get();
-
-            failed = 
-                    (getConfig(USE_SERVICE_STATE_RUNNING) && status == Lifecycle.ON_FIRE && !weSetItOnFire) ||
-                    (Boolean.FALSE.equals(isUp) &&
-                            (getConfig(USE_SERVICE_STATE_RUNNING) ? status == Lifecycle.RUNNING : true) && 
-                            (getConfig(ONLY_REPORT_IF_PREVIOUSLY_UP) ? lastUpTime != null : true));
-            healthy = 
-                    (getConfig(USE_SERVICE_STATE_RUNNING) ? (status == Lifecycle.RUNNING || (weSetItOnFire && status == Lifecycle.ON_FIRE)) : 
-                        true) && 
-                    Boolean.TRUE.equals(isUp);
-        }
-        
-        public String getDescription() {
-            Long lastUpTime = serviceLastUp.get();
-            Boolean isUp = serviceIsUp.get();
-            Lifecycle status = serviceState.get();
-            Duration serviceFailedStabilizationDelay = getServiceFailedStabilizationDelay();
-            Duration serviceRecoveredStabilizationDelay = getServiceRecoveredStabilizationDelay();
-
-            return String.format("location=%s; isUp=%s; status=%s; timeNow=%s; lastReportedUp=%s; lastPublished=%s; "+
-                        "currentFailurePeriod=%s; currentRecoveryPeriod=%s",
-                    entity.getLocations(), 
-                    (isUp != null ? isUp : "<unreported>"),
-                    (status != null ? status : "<unreported>"),
-                    Time.makeDateString(System.currentTimeMillis()),
-                    (lastUpTime != null ? Time.makeDateString(lastUpTime) : "<never>"),
-                    lastPublished,
-                    (currentFailureStartTime != null ? getTimeStringSince(currentFailureStartTime) : "<none>") + " (stabilization "+makeTimeStringRounded(serviceFailedStabilizationDelay) + ")",
-                    (currentRecoveryStartTime != null ? getTimeStringSince(currentRecoveryStartTime) : "<none>") + " (stabilization "+makeTimeStringRounded(serviceRecoveredStabilizationDelay) + ")");
-        }
-    }
-    
-    private long getTimeDiff(Long recent, Long previous) {
-        return (previous == null) ? recent : (recent - previous);
-    }
-    
-    private String getTimeStringSince(Long time) {
-        return time == null ? null : Time.makeTimeStringRounded(System.currentTimeMillis() - time);
-    }
-    
-    private String getTimeStringSince(AtomicReference<Long> timeRef) {
-        return getTimeStringSince(timeRef.get());
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java
----------------------------------------------------------------------
diff --git a/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java b/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java
index 39be9d4..319b345 100644
--- a/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java
+++ b/policy/src/test/java/brooklyn/entity/brooklyn/BrooklynMetricsTest.java
@@ -26,22 +26,24 @@ import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import brooklyn.entity.Entity;
-import brooklyn.entity.basic.ApplicationBuilder;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.event.AttributeSensor;
 import brooklyn.event.SensorEventListener;
 import brooklyn.location.basic.SimulatedLocation;
 import brooklyn.test.Asserts;
-import brooklyn.test.entity.LocalManagementContextForTests;
 import brooklyn.test.entity.TestApplication;
 import brooklyn.test.entity.TestEntity;
+import brooklyn.test.entity.TestEntityNoEnrichersImpl;
+import brooklyn.util.collections.MutableMap;
+import brooklyn.util.time.Duration;
 
 import com.google.common.collect.ImmutableList;
 
 public class BrooklynMetricsTest {
 
     private static final long TIMEOUT_MS = 2*1000;
+    private final static int DEFAULT_SUBSCRIPTIONS_PER_ENTITY = 2;
     
     TestApplication app;
     SimulatedLocation loc;
@@ -52,7 +54,6 @@ public class BrooklynMetricsTest {
         loc = new SimulatedLocation();
         app = TestApplication.Factory.newManagedInstanceForTests();
         brooklynMetrics = app.createAndManageChild(EntitySpec.create(BrooklynMetrics.class).configure("updatePeriod", 10L));
-        Entities.manage(brooklynMetrics);
     }
     
     @AfterMethod(alwaysRun=true)
@@ -64,7 +65,7 @@ public class BrooklynMetricsTest {
     public void testInitialBrooklynMetrics() {
         app.start(ImmutableList.of(loc));
 
-        Asserts.succeedsEventually(new Runnable() {
+        Asserts.succeedsEventually(MutableMap.of("timeout", Duration.FIVE_SECONDS), new Runnable() {
             public void run() {
                 assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EFFECTORS_INVOKED), (Long)1L);
                 assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_TASKS_SUBMITTED) > 0);
@@ -72,16 +73,16 @@ public class BrooklynMetricsTest {
                 assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_ACTIVE_TASKS), (Long)0L);
                 assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_PUBLISHED) > 0);
                 assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_DELIVERED), (Long)0L);
-                assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long)0L);
+                assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long)(2L*DEFAULT_SUBSCRIPTIONS_PER_ENTITY));
             }});
     }
     
     @Test
     public void testBrooklynMetricsIncremented() {
-        TestEntity e = app.createAndManageChild(EntitySpec.create(TestEntity.class));
+        TestEntity e = app.createAndManageChild(EntitySpec.create(TestEntity.class, TestEntityNoEnrichersImpl.class));
         app.start(ImmutableList.of(loc));
 
-        Asserts.succeedsEventually(new Runnable() {
+        Asserts.succeedsEventually(MutableMap.of("timeout", Duration.FIVE_SECONDS), new Runnable() {
             public void run() {
                 assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EFFECTORS_INVOKED), (Long)2L); // for app and testEntity's start
             }});
@@ -106,11 +107,12 @@ public class BrooklynMetricsTest {
         app.subscribe(e, TestEntity.SEQUENCE, SensorEventListener.NOOP);
         e.setAttribute(TestEntity.SEQUENCE, 1);
         
-        Asserts.succeedsEventually(new Runnable() {
+        Asserts.succeedsEventually(MutableMap.of("timeout", Duration.FIVE_SECONDS), new Runnable() {
             public void run() {
                 assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_PUBLISHED) > eventsPublished);
                 assertTrue(brooklynMetrics.getAttribute(BrooklynMetrics.TOTAL_EVENTS_DELIVERED) > eventsDelivered);
-                assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long)1L);
+                assertEquals(brooklynMetrics.getAttribute(BrooklynMetrics.NUM_SUBSCRIPTIONS), (Long)
+                    (1L + 2*DEFAULT_SUBSCRIPTIONS_PER_ENTITY));
             }});
     }
     

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java
----------------------------------------------------------------------
diff --git a/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java b/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java
index 2290a33..7cec2ed 100644
--- a/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java
+++ b/policy/src/test/java/brooklyn/policy/autoscaling/AutoScalerPolicyMetricTest.java
@@ -31,7 +31,6 @@ import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import brooklyn.entity.basic.ApplicationBuilder;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.event.AttributeSensor;
@@ -49,7 +48,7 @@ import com.google.common.collect.Lists;
 public class AutoScalerPolicyMetricTest {
     
     private static long TIMEOUT_MS = 10000;
-    private static long SHORT_WAIT_MS = 250;
+    private static long SHORT_WAIT_MS = 50;
     
     private static final AttributeSensor<Integer> MY_ATTRIBUTE = Sensors.newIntegerSensor("autoscaler.test.intAttrib");
     TestApplication app;
@@ -57,7 +56,7 @@ public class AutoScalerPolicyMetricTest {
     
     @BeforeMethod(alwaysRun=true)
     public void before() {
-        app = ApplicationBuilder.newManagedApp(TestApplication.class);
+        app = TestApplication.Factory.newManagedInstanceForTests();
         tc = app.createAndManageChild(EntitySpec.create(TestCluster.class)
                 .configure("initialSize", 1));
     }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java
----------------------------------------------------------------------
diff --git a/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java b/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java
index 9d8f427..0cccf14 100644
--- a/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java
+++ b/policy/src/test/java/brooklyn/policy/ha/HaPolicyRebindTest.java
@@ -31,6 +31,7 @@ import org.testng.annotations.Test;
 import brooklyn.entity.Entity;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.Lifecycle;
+import brooklyn.entity.basic.ServiceStateLogic;
 import brooklyn.entity.group.DynamicCluster;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.entity.rebind.RebindTestFixtureWithApp;
@@ -40,6 +41,7 @@ import brooklyn.event.SensorEventListener;
 import brooklyn.location.Location;
 import brooklyn.location.LocationSpec;
 import brooklyn.location.basic.SimulatedLocation;
+import brooklyn.policy.EnricherSpec;
 import brooklyn.policy.PolicySpec;
 import brooklyn.policy.ha.HASensors.FailureDescriptor;
 import brooklyn.test.Asserts;
@@ -131,7 +133,7 @@ public class HaPolicyRebindTest extends RebindTestFixtureWithApp {
     
     @Test
     public void testServiceFailureDetectorWorksAfterRebind() throws Exception {
-        origEntity.addPolicy(PolicySpec.create(ServiceFailureDetector.class));
+        origEntity.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
 
         // rebind
         TestApplication newApp = rebind();
@@ -139,9 +141,10 @@ public class HaPolicyRebindTest extends RebindTestFixtureWithApp {
 
         newApp.getManagementContext().getSubscriptionManager().subscribe(newEntity, HASensors.ENTITY_FAILED, eventListener);
 
-        // stimulate the policy
-        newEntity.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
         newEntity.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(newEntity, Lifecycle.RUNNING);
+        
+        // trigger the failure
         newEntity.setAttribute(TestEntity.SERVICE_UP, false);
 
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(newEntity), null);

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java
----------------------------------------------------------------------
diff --git a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java
index 3426598..4649b36 100644
--- a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java
+++ b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorStabilizationTest.java
@@ -31,12 +31,14 @@ import org.testng.annotations.Test;
 import brooklyn.entity.basic.ApplicationBuilder;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.Lifecycle;
+import brooklyn.entity.basic.ServiceStateLogic;
+import brooklyn.entity.basic.ServiceStateLogicTest;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.event.Sensor;
 import brooklyn.event.SensorEvent;
 import brooklyn.event.SensorEventListener;
 import brooklyn.management.ManagementContext;
-import brooklyn.policy.PolicySpec;
+import brooklyn.policy.EnricherSpec;
 import brooklyn.policy.ha.HASensors.FailureDescriptor;
 import brooklyn.test.Asserts;
 import brooklyn.test.entity.LocalManagementContextForTests;
@@ -49,6 +51,7 @@ import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableMap;
 
+/** also see more primitive tests in {@link ServiceStateLogicTest} */
 public class ServiceFailureDetectorStabilizationTest {
 
     private static final int TIMEOUT_MS = 10*1000;
@@ -67,8 +70,8 @@ public class ServiceFailureDetectorStabilizationTest {
         managementContext = new LocalManagementContextForTests();
         app = ApplicationBuilder.newManagedApp(TestApplication.class, managementContext);
         e1 = app.createAndManageChild(EntitySpec.create(TestEntity.class));
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
         
         app.getManagementContext().getSubscriptionManager().subscribe(
                 e1, 
@@ -95,8 +98,8 @@ public class ServiceFailureDetectorStabilizationTest {
     
     @Test(groups="Integration") // Because slow
     public void testNotNotifiedOfTemporaryFailuresDuringStabilisationDelay() throws Exception {
-        e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class)
-                .configure(ServiceFailureDetector.SERVICE_FAILED_STABILIZATION_DELAY, Duration.ONE_MINUTE));
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+                .configure(ServiceFailureDetector.ENTITY_FAILED_STABILIZATION_DELAY, Duration.ONE_MINUTE));
         
         e1.setAttribute(TestEntity.SERVICE_UP, false);
         Thread.sleep(100);
@@ -109,8 +112,8 @@ public class ServiceFailureDetectorStabilizationTest {
     public void testNotifiedOfFailureAfterStabilisationDelay() throws Exception {
         final int stabilisationDelay = 1000;
         
-        e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class)
-                .configure(ServiceFailureDetector.SERVICE_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+                .configure(ServiceFailureDetector.ENTITY_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
         
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
@@ -122,8 +125,8 @@ public class ServiceFailureDetectorStabilizationTest {
     public void testFailuresThenUpDownResetsStabilisationCount() throws Exception {
         final long stabilisationDelay = 1000;
         
-        e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class)
-                .configure(ServiceFailureDetector.SERVICE_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+                .configure(ServiceFailureDetector.ENTITY_FAILED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
         
         e1.setAttribute(TestEntity.SERVICE_UP, false);
         assertNoEventsContinually(Duration.of(stabilisationDelay - OVERHEAD));
@@ -139,8 +142,8 @@ public class ServiceFailureDetectorStabilizationTest {
     public void testNotNotifiedOfTemporaryRecoveryDuringStabilisationDelay() throws Exception {
         final long stabilisationDelay = 1000;
         
-        e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class)
-                .configure(ServiceFailureDetector.SERVICE_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+                .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
         
         e1.setAttribute(TestEntity.SERVICE_UP, false);
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
@@ -157,8 +160,8 @@ public class ServiceFailureDetectorStabilizationTest {
     public void testNotifiedOfRecoveryAfterStabilisationDelay() throws Exception {
         final int stabilisationDelay = 1000;
         
-        e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class)
-                .configure(ServiceFailureDetector.SERVICE_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+                .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
         
         e1.setAttribute(TestEntity.SERVICE_UP, false);
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
@@ -173,8 +176,8 @@ public class ServiceFailureDetectorStabilizationTest {
     public void testRecoversThenDownUpResetsStabilisationCount() throws Exception {
         final long stabilisationDelay = 1000;
         
-        e1.addPolicy(PolicySpec.create(ServiceFailureDetector.class)
-                .configure(ServiceFailureDetector.SERVICE_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+                .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.of(stabilisationDelay)));
         
         e1.setAttribute(TestEntity.SERVICE_UP, false);
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java
----------------------------------------------------------------------
diff --git a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java
index 98c28de..5c9dd66 100644
--- a/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java
+++ b/policy/src/test/java/brooklyn/policy/ha/ServiceFailureDetectorTest.java
@@ -32,11 +32,14 @@ import org.testng.annotations.Test;
 import brooklyn.entity.basic.ApplicationBuilder;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.Lifecycle;
+import brooklyn.entity.basic.ServiceStateLogic;
+import brooklyn.entity.basic.ServiceStateLogic.ServiceProblemsLogic;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.event.Sensor;
 import brooklyn.event.SensorEvent;
 import brooklyn.event.SensorEventListener;
 import brooklyn.management.ManagementContext;
+import brooklyn.policy.EnricherSpec;
 import brooklyn.policy.ha.HASensors.FailureDescriptor;
 import brooklyn.test.Asserts;
 import brooklyn.test.EntityTestUtils;
@@ -44,19 +47,19 @@ import brooklyn.test.entity.LocalManagementContextForTests;
 import brooklyn.test.entity.TestApplication;
 import brooklyn.test.entity.TestEntity;
 import brooklyn.util.collections.MutableMap;
+import brooklyn.util.time.Duration;
+import brooklyn.util.time.Time;
 
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableMap;
 
 public class ServiceFailureDetectorTest {
 
-    private static final int TIMEOUT_MS = 10*1000;
+    private static final int TIMEOUT_MS = 1*1000;
 
     private ManagementContext managementContext;
     private TestApplication app;
     private TestEntity e1;
-    private ServiceFailureDetector policy;
     
     private List<SensorEvent<FailureDescriptor>> events;
     private SensorEventListener<FailureDescriptor> eventListener;
@@ -73,6 +76,7 @@ public class ServiceFailureDetectorTest {
         managementContext = new LocalManagementContextForTests();
         app = ApplicationBuilder.newManagedApp(TestApplication.class, managementContext);
         e1 = app.createAndManageChild(EntitySpec.create(TestEntity.class));
+        e1.addEnricher(ServiceStateLogic.newEnricherForServiceStateFromProblemsAndUp());
         
         app.getManagementContext().getSubscriptionManager().subscribe(e1, HASensors.ENTITY_FAILED, eventListener);
         app.getManagementContext().getSubscriptionManager().subscribe(e1, HASensors.ENTITY_RECOVERED, eventListener);
@@ -86,181 +90,220 @@ public class ServiceFailureDetectorTest {
     @Test(groups="Integration") // Has a 1 second wait
     public void testNotNotifiedOfFailuresForHealthy() throws Exception {
         // Create members before and after the policy is registered, to test both scenarios
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
         
-        policy = new ServiceFailureDetector();
-        e1.addPolicy(policy);
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
         
         assertNoEventsContinually();
+        assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING);
     }
     
     @Test
     public void testNotifiedOfFailure() throws Exception {
-        policy = new ServiceFailureDetector();
-        e1.addPolicy(policy);
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
         
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
+        
+        assertEquals(events.size(), 0, "events="+events);
+        
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
         assertEquals(events.size(), 1, "events="+events);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
     }
     
     @Test
-    public void testNotifiedOfFailureOnStateOnFire() throws Exception {
-        policy = new ServiceFailureDetector();
-        e1.addPolicy(policy);
+    public void testNotifiedOfFailureOnProblem() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
+        
+        e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
         
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.ON_FIRE);
+        assertEquals(events.size(), 0, "events="+events);
+        
+        ServiceProblemsLogic.updateProblemsIndicator(e1, "test", "foo");
 
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
         assertEquals(events.size(), 1, "events="+events);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
+    }
+    
+    @Test
+    public void testNotifiedOfFailureOnStateOnFire() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
+        e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.ON_FIRE);
+
+        assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
+        assertEquals(events.size(), 1, "events="+events);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
     }
     
     @Test
     public void testNotifiedOfRecovery() throws Exception {
-        policy = new ServiceFailureDetector();
-        e1.addPolicy(policy);
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
         
-        // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
+        // Make the entity fail
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
 
         // And make the entity recover
         e1.setAttribute(TestEntity.SERVICE_UP, true);
         assertHasEventEventually(HASensors.ENTITY_RECOVERED, Predicates.<Object>equalTo(e1), null);
         assertEquals(events.size(), 2, "events="+events);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING);
     }
     
-    @Test(groups="Integration") // Has a 1 second wait
-    public void testOnlyReportsFailureIfPreviouslyUp() throws Exception {
-        policy = new ServiceFailureDetector();
-        e1.addPolicy(policy);
+    @Test
+    public void testNotifiedOfRecoveryFromProblems() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
         
+        e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
         // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
-        e1.setAttribute(TestEntity.SERVICE_UP, false);
+        ServiceProblemsLogic.updateProblemsIndicator(e1, "test", "foo");
 
-        assertNoEventsContinually();
+        assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
+
+        // And make the entity recover
+        ServiceProblemsLogic.clearProblemsIndicator(e1, "test");
+        assertHasEventEventually(HASensors.ENTITY_RECOVERED, Predicates.<Object>equalTo(e1), null);
+        assertEquals(events.size(), 2, "events="+events);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING);
     }
     
-    @Test
-    public void testDisablingOnlyReportsFailureIfPreviouslyUp() throws Exception {
-        policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
+    
+    @Test(groups="Integration") // Has a 1 second wait
+    public void testEmitsEntityFailureOnlyIfPreviouslyUp() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
         
         // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, false);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
 
-        assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
+        assertNoEventsContinually();
     }
     
     @Test
-    public void testSetsOnFireOnFailure() throws Exception {
-        policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
+    public void testDisablingPreviouslyUpRequirementForEntityFailed() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+            .configure(ServiceFailureDetector.ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP, false));
         
-        // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, false);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
 
-        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE, Lifecycle.ON_FIRE);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
+        assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
     }
     
     @Test
     public void testDisablingSetsOnFireOnFailure() throws Exception {
-        policy = new ServiceFailureDetector(ImmutableMap.of("setOnFireOnFailure", false, "onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+            .configure(ServiceFailureDetector.SERVICE_ON_FIRE_STABILIZATION_DELAY, Duration.PRACTICALLY_FOREVER));
         
         // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
+        e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
-        EntityTestUtils.assertAttributeEqualsContinually(e1, TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
+        assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING);
     }
     
     @Test(groups="Integration") // Has a 1 second wait
-    public void testUsesServiceStateRunning() throws Exception {
-        policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
+    public void testOnFireAfterDelay() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+            .configure(ServiceFailureDetector.SERVICE_ON_FIRE_STABILIZATION_DELAY, Duration.ONE_SECOND));
         
-        // entity no counted as failed, because serviceState != running || onfire
+        // Make the entity fail
+        e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
-        assertNoEventsContinually();
+        assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING);
+        Time.sleep(Duration.millis(100));
+        assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING);
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
     }
-
-    @Test
-    public void testDisablingUsesServiceStateRunning() throws Exception {
-        policy = new ServiceFailureDetector(ImmutableMap.of("useServiceStateRunning", false, "onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
+    
+    @Test(groups="Integration") // Has a 1 second wait
+    public void testOnFailureDelayFromProblemAndRecover() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+            .configure(ServiceFailureDetector.SERVICE_ON_FIRE_STABILIZATION_DELAY, Duration.ONE_SECOND)
+            .configure(ServiceFailureDetector.ENTITY_RECOVERED_STABILIZATION_DELAY, Duration.ONE_SECOND));
         
         // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_UP, false);
+        e1.setAttribute(TestEntity.SERVICE_UP, true);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
 
+        assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING);
+        ServiceStateLogic.ServiceProblemsLogic.updateProblemsIndicator(e1, "test", "foo");
+        
+        assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING);
+        Time.sleep(Duration.millis(100));
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
+        assertEquals(e1.getAttribute(TestEntity.SERVICE_STATE_ACTUAL), Lifecycle.RUNNING);
+        
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
+        
+        // Now recover
+        ServiceStateLogic.ServiceProblemsLogic.clearProblemsIndicator(e1, "test");
+        EntityTestUtils.assertAttributeEqualsEventually(e1, TestEntity.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING);
+        
+        assertEquals(events.size(), 1, "events="+events);
+        
+        assertHasEventEventually(HASensors.ENTITY_RECOVERED, Predicates.<Object>equalTo(e1), null);
+        assertEquals(events.size(), 2, "events="+events);
     }
-
+    
     @Test(groups="Integration") // Has a 1 second wait
-    public void testOnlyReportsFailureIfRunning() throws Exception {
-        policy = new ServiceFailureDetector();
-        e1.addPolicy(policy);
+    public void testAttendsToServiceState() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
         
-        // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.STARTING);
         e1.setAttribute(TestEntity.SERVICE_UP, true);
+        // not counted as failed because not expected to be running
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
         assertNoEventsContinually();
     }
-    
-    @Test
-    public void testReportsFailureWhenNotPreviouslyUp() throws Exception {
-        policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
-        
-        // Make the entity fail
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
-        e1.setAttribute(TestEntity.SERVICE_UP, false);
 
-        assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
-    }
-    
-    @Test
-    public void testReportsFailureWhenNoServiceState() throws Exception {
-        policy = new ServiceFailureDetector(ImmutableMap.of("useServiceStateRunning", false));
-        e1.addPolicy(policy);
+    @Test(groups="Integration") // Has a 1 second wait
+    public void testOnlyReportsFailureIfRunning() throws Exception {
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class));
         
         // Make the entity fail
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.STARTING);
         e1.setAttribute(TestEntity.SERVICE_UP, true);
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
-        assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
+        assertNoEventsContinually();
     }
     
     @Test
     public void testReportsFailureWhenAlreadyDownOnRegisteringPolicy() throws Exception {
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.RUNNING);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.RUNNING);
         e1.setAttribute(TestEntity.SERVICE_UP, false);
 
-        policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+            .configure(ServiceFailureDetector.ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP, false));
 
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
     }
     
     @Test
     public void testReportsFailureWhenAlreadyOnFireOnRegisteringPolicy() throws Exception {
-        e1.setAttribute(TestEntity.SERVICE_STATE, Lifecycle.ON_FIRE);
+        ServiceStateLogic.setExpectedState(e1, Lifecycle.ON_FIRE);
 
-        policy = new ServiceFailureDetector(ImmutableMap.of("onlyReportIfPreviouslyUp", false));
-        e1.addPolicy(policy);
+        e1.addEnricher(EnricherSpec.create(ServiceFailureDetector.class)
+            .configure(ServiceFailureDetector.ENTITY_FAILED_ONLY_IF_PREVIOUSLY_UP, false));
 
         assertHasEventEventually(HASensors.ENTITY_FAILED, Predicates.<Object>equalTo(e1), null);
     }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b2daedf8/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java
----------------------------------------------------------------------
diff --git a/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java b/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java
index 4b5a0d9..8d1a683 100644
--- a/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java
+++ b/policy/src/test/java/brooklyn/policy/ha/ServiceReplacerTest.java
@@ -138,7 +138,7 @@ public class ServiceReplacerTest {
         e1.emit(HASensors.ENTITY_FAILED, new FailureDescriptor(e1, "simulate failure"));
         
         // Expect cluster to go on-fire when fails to start replacement
-        EntityTestUtils.assertAttributeEqualsEventually(cluster, Attributes.SERVICE_STATE, Lifecycle.ON_FIRE);
+        EntityTestUtils.assertAttributeEqualsEventually(cluster, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.ON_FIRE);
         
         // And expect to have the second failed entity still kicking around as proof (in quarantine)
         Iterable<Entity> members = Iterables.filter(managementContext.getEntityManager().getEntities(), Predicates.instanceOf(FailingEntity.class));


Mime
View raw message