brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aleds...@apache.org
Subject [2/3] brooklyn-server git commit: DynamicMultiGroup: fix/test rebind
Date Fri, 25 Mar 2016 16:23:14 GMT
DynamicMultiGroup: fix/test rebind

Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/67a724e6
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/67a724e6
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/67a724e6

Branch: refs/heads/master
Commit: 67a724e655f33188172c7a5784e98cde809a48ee
Parents: 7d7c1a8
Author: Aled Sage <aled.sage@gmail.com>
Authored: Fri Mar 25 15:19:32 2016 +0000
Committer: Aled Sage <aled.sage@gmail.com>
Committed: Fri Mar 25 15:19:32 2016 +0000

----------------------------------------------------------------------
 .../entity/group/DynamicMultiGroupImpl.java     |  38 +++++-
 .../group/DynamicMultiGroupRebindTest.java      | 120 +++++++++++++++++++
 .../entity/group/DynamicMultiGroupTest.java     |  28 +----
 3 files changed, 159 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/67a724e6/core/src/main/java/org/apache/brooklyn/entity/group/DynamicMultiGroupImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/brooklyn/entity/group/DynamicMultiGroupImpl.java
b/core/src/main/java/org/apache/brooklyn/entity/group/DynamicMultiGroupImpl.java
index 5c507a3..44dcd10 100644
--- a/core/src/main/java/org/apache/brooklyn/entity/group/DynamicMultiGroupImpl.java
+++ b/core/src/main/java/org/apache/brooklyn/entity/group/DynamicMultiGroupImpl.java
@@ -28,11 +28,15 @@ import javax.annotation.Nullable;
 import org.apache.brooklyn.api.entity.Entity;
 import org.apache.brooklyn.api.entity.EntitySpec;
 import org.apache.brooklyn.api.entity.Group;
+import org.apache.brooklyn.api.mgmt.ha.ManagementNodeState;
 import org.apache.brooklyn.api.sensor.AttributeSensor;
 import org.apache.brooklyn.core.entity.Entities;
 import org.apache.brooklyn.feed.function.FunctionFeed;
 import org.apache.brooklyn.feed.function.FunctionPollConfig;
 import org.apache.brooklyn.util.collections.MutableMap;
+import org.apache.brooklyn.util.time.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
@@ -46,6 +50,8 @@ import com.google.common.collect.Sets;
 
 public class DynamicMultiGroupImpl extends DynamicGroupImpl implements DynamicMultiGroup
{
 
+    private static final Logger LOG = LoggerFactory.getLogger(DynamicMultiGroupImpl.class);
+
     /**
      * {@link Function} for deriving bucket names from a sensor value.
      */
@@ -119,7 +125,36 @@ public class DynamicMultiGroupImpl extends DynamicGroupImpl implements
DynamicMu
         super.rebind();
 
         if (rescan == null) {
-            connectScanner();
+            // The rescan can (in a different thread) cause us to remove the empty groups
- i.e. remove 
+            // it as a child, and unmanage it. That is dangerous during rebind, because the
rebind-thread 
+            // may concurrently (or subsequently) be initialising that child entity. It has
caused 
+            // rebind errors where the child's entity-rebind complains in setParent() that
it was
+            // "previouslyOwned". Therefore we defer registering/executing our scanner until
rebind is
+            // complete, so all entities are reconstituted.
+            // We don't worry about other managementNodeStates, such as standby: if we were
told to
+            // rebind then we are free to fully initialise ourselves. But we do double-check
that we
+            // are still managed before trying to execute.
+            
+            getExecutionContext().execute(new Runnable() {
+                @Override public void run() {
+                    LOG.debug("Deferring scanner for {} until management context initialisation
complete", DynamicMultiGroupImpl.this);
+                    while (!isRebindComplete()) {
+                        Time.sleep(100); // avoid thrashing
+                    }
+                    LOG.debug("Connecting scanner for {}", DynamicMultiGroupImpl.this);
+                    connectScanner();
+                }
+                private boolean isRebindComplete() {
+                    // TODO Want to determine if finished rebinding (either success or fail
is fine).
+                    // But not a clean way to do this that works for both unit tests and
live server?!
+                    //  * In RebindTestFixtureWithApp tests, mgmt.getHighAvailabilityManager().getNodeState()
+                    //    always returns INITIALIZING.
+                    //  * The rebind metrics is a hack, and feels very risky for HOT_STANDBY
nodes that 
+                    //    may have executed the rebind code multiple times.
+                    Map<String, Object> metrics = getManagementContext().getRebindManager().getMetrics();
+                    Object count = (metrics.get("rebind") instanceof Map) ? ((Map<?,?>)metrics.get("rebind")).get("count")
: null;
+                    return (count instanceof Number) && ((Number)count).intValue()
> 0;
+                }});
         }
     }
 
@@ -190,6 +225,7 @@ public class DynamicMultiGroupImpl extends DynamicGroupImpl implements
DynamicMu
             Set<String> empty = ImmutableSet.copyOf(Sets.difference(buckets.keySet(),
entityMapping.keySet()));
             for (String name : empty) {
                 Group removed = buckets.remove(name);
+                LOG.debug(this+" removing empty child-bucket "+name+" -> "+removed);
                 removeChild(removed);
                 Entities.unmanage(removed);
             }

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/67a724e6/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupRebindTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupRebindTest.java
b/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupRebindTest.java
new file mode 100644
index 0000000..9f2a302
--- /dev/null
+++ b/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupRebindTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.brooklyn.entity.group;
+
+import static com.google.common.base.Predicates.instanceOf;
+import static com.google.common.collect.Iterables.find;
+import static org.apache.brooklyn.core.entity.EntityPredicates.displayNameEqualTo;
+import static org.apache.brooklyn.entity.group.DynamicMultiGroup.BUCKET_FUNCTION;
+import static org.apache.brooklyn.entity.group.DynamicMultiGroupImpl.bucketFromAttribute;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
+import java.util.List;
+
+import org.apache.brooklyn.api.entity.EntitySpec;
+import org.apache.brooklyn.api.entity.Group;
+import org.apache.brooklyn.api.sensor.AttributeSensor;
+import org.apache.brooklyn.core.entity.EntityPredicates;
+import org.apache.brooklyn.core.mgmt.rebind.RebindOptions;
+import org.apache.brooklyn.core.mgmt.rebind.RebindTestFixtureWithApp;
+import org.apache.brooklyn.core.sensor.Sensors;
+import org.apache.brooklyn.core.test.entity.TestEntity;
+import org.apache.brooklyn.test.Asserts;
+import org.apache.brooklyn.util.time.Duration;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+
+public class DynamicMultiGroupRebindTest extends RebindTestFixtureWithApp {
+
+    private static final AttributeSensor<String> SENSOR = Sensors.newSensor(String.class,
"multigroup.test");
+
+    // Previously there was a bug on rebind. The entity's rebind would immediately connec
the 
+    // rescan, which would start executing in another thread. If there were any empty buckets
+    // (i.e. groups) that child would be removed. But the rebind-manager would still be executing
+    // concurrently. The empty group that was being removed might not have been reconstituted
yet.
+    // When we then tried to reconstitute it, the abstractEntity.setParent would fail with
an 
+    // error about being previouslyOwned, causing rebind to fail.
+    //
+    // To recreate this error, need to have several apps concurrently so that the rebind
order
+    // of the entities will be interleaved.
+    @Test(groups="Integration", invocationCount=10)
+    public void testRebindWhenGroupDisappeared() throws Exception {
+        int NUM_ITERATIONS = 10;
+        List<DynamicMultiGroup> dmgs = Lists.newArrayList();
+        List<TestEntity> childs = Lists.newArrayList();
+        
+        // Create lots of DynamicMultiGroups - one entity for each
+        for (int i = 0; i < NUM_ITERATIONS; i++) {
+            Group group = origApp.createAndManageChild(EntitySpec.create(BasicGroup.class));
+            DynamicMultiGroup dmg = origApp.createAndManageChild(EntitySpec.create(DynamicMultiGroup.class)
+                    .displayName("dmg"+i)
+                    .configure(DynamicMultiGroup.ENTITY_FILTER, Predicates.and(EntityPredicates.displayNameEqualTo("child"+i),
instanceOf(TestEntity.class)))
+                    .configure(DynamicMultiGroup.RESCAN_INTERVAL, 5L)
+                    .configure(BUCKET_FUNCTION, bucketFromAttribute(SENSOR))
+                    .configure(DynamicMultiGroup.BUCKET_SPEC, EntitySpec.create(BasicGroup.class)));
+            dmgs.add(dmg);
+            
+            TestEntity child = group.addChild(EntitySpec.create(TestEntity.class).displayName("child"+i));
+            child.sensors().set(SENSOR, "bucketA");
+            childs.add(child);
+        }
+        
+        // Wait for all DynamicMultiGroups to have initialised correctly
+        for (int i = 0; i < NUM_ITERATIONS; i++) {
+            final DynamicMultiGroup dmg = dmgs.get(i);
+            final TestEntity child = childs.get(i);
+            Asserts.succeedsEventually(new Runnable() {
+                public void run() {
+                    Group bucketA = (Group) find(dmg.getChildren(), displayNameEqualTo("bucketA"),
null);
+                    assertNotNull(bucketA);
+                    assertEquals(ImmutableSet.copyOf(bucketA.getMembers()), ImmutableSet.of(child));
+                }
+            });
+        }
+
+        // Quickly change the child sensors, and shutdown immediately before the DynamicMultiGroups

+        // rescan and update themselves.
+        for (int i = 0; i < NUM_ITERATIONS; i++) {
+            childs.get(i).sensors().set(SENSOR, "bucketB");
+        }
+        rebind(RebindOptions.create().terminateOrigManagementContext(true));
+        
+        // Check that all entities are in the new expected groups
+        for (int i = 0; i < NUM_ITERATIONS; i++) {
+            final DynamicMultiGroup dmg = (DynamicMultiGroup) newManagementContext.getEntityManager().getEntity(dmgs.get(i).getId());
+            final TestEntity child = (TestEntity) newManagementContext.getEntityManager().getEntity(childs.get(i).getId());
+            // FIXME Remove timeout; use default
+            Asserts.succeedsEventually(new Runnable() {
+                public void run() {
+                    Group bucketA = (Group) find(dmg.getChildren(), displayNameEqualTo("bucketA"),
null);
+                    Group bucketB = (Group) find(dmg.getChildren(), displayNameEqualTo("bucketB"),
null);
+                    assertNull(bucketA);
+                    assertNotNull(bucketB);
+                    assertEquals(ImmutableSet.copyOf(bucketB.getMembers()), ImmutableSet.of(child));
+                }
+            });
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/67a724e6/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupTest.java
b/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupTest.java
index b156bb5..efa3981 100644
--- a/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupTest.java
+++ b/core/src/test/java/org/apache/brooklyn/entity/group/DynamicMultiGroupTest.java
@@ -34,42 +34,18 @@ import org.apache.brooklyn.api.entity.Group;
 import org.apache.brooklyn.api.sensor.AttributeSensor;
 import org.apache.brooklyn.api.sensor.SensorEvent;
 import org.apache.brooklyn.api.sensor.SensorEventListener;
-import org.apache.brooklyn.core.entity.Entities;
-import org.apache.brooklyn.core.entity.factory.ApplicationBuilder;
-import org.apache.brooklyn.core.location.SimulatedLocation;
 import org.apache.brooklyn.core.sensor.Sensors;
-import org.apache.brooklyn.core.test.entity.LocalManagementContextForTests;
-import org.apache.brooklyn.core.test.entity.TestApplication;
+import org.apache.brooklyn.core.test.BrooklynAppUnitTestSupport;
 import org.apache.brooklyn.core.test.entity.TestEntity;
-import org.apache.brooklyn.entity.group.BasicGroup;
-import org.apache.brooklyn.entity.group.DynamicMultiGroup;
 import org.apache.brooklyn.test.Asserts;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 
-public class DynamicMultiGroupTest {
+public class DynamicMultiGroupTest extends BrooklynAppUnitTestSupport {
 
     private static final AttributeSensor<String> SENSOR = Sensors.newSensor(String.class,
"multigroup.test");
 
-    private TestApplication app;
-
-
-    @BeforeMethod(alwaysRun = true)
-    public void setUp() {
-        app = ApplicationBuilder.newManagedApp(TestApplication.class, new LocalManagementContextForTests());
-        app.start(ImmutableList.of(new SimulatedLocation()));
-    }
-
-    @AfterMethod(alwaysRun = true)
-    public void tearDown(){
-        if (app != null)
-            Entities.destroyAll(app.getManagementContext());
-    }
-
     @Test
     public void testBucketDistributionFromSubscription() {
         Group group = app.createAndManageChild(EntitySpec.create(BasicGroup.class));


Mime
View raw message