brooklyn-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rich...@apache.org
Subject [06/10] git commit: tests for volumes read and written, plus conveniences for cleaning
Date Tue, 10 Jun 2014 00:15:27 GMT
tests for volumes read and written, plus conveniences for cleaning


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/94495f16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/94495f16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/94495f16

Branch: refs/heads/master
Commit: 94495f16340fa82eed2a1bcc19c5c1493023d7d3
Parents: 522d339
Author: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Authored: Thu Jun 5 16:09:16 2014 +0100
Committer: Alex Heneveld <alex.heneveld@cloudsoftcorp.com>
Committed: Thu Jun 5 16:09:16 2014 +0100

----------------------------------------------------------------------
 .../rebind/persister/InMemoryObjectStore.java   |   4 +-
 ...ntoPersisterInMemorySizeIntegrationTest.java |  75 ++++++++
 .../BrooklynMementoPersisterInMemoryTest.java   |   8 -
 .../rebind/persister/ListeningObjectStore.java  | 185 +++++++++++++++++++
 .../persister/jclouds/BlobStoreCleaner.java     |  54 ++++++
 5 files changed, 316 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
b/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
index 85fd1b0..b89d611 100644
--- a/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
+++ b/core/src/main/java/brooklyn/entity/rebind/persister/InMemoryObjectStore.java
@@ -22,11 +22,11 @@ public class InMemoryObjectStore implements PersistenceObjectStore {
 
     Map<String,String> filesByName = MutableMap.<String,String>of();
     boolean prepared = false;
-
+    
     public InMemoryObjectStore() {
         log.info("Using memory-based objectStore");
     }
-
+    
     @Override
     public String getSummaryName() {
         return "in-memory (test) persistence store";

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java
b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java
new file mode 100644
index 0000000..bd7fa04
--- /dev/null
+++ b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemorySizeIntegrationTest.java
@@ -0,0 +1,75 @@
+package brooklyn.entity.rebind.persister;
+
+import java.io.IOException;
+import java.util.concurrent.TimeoutException;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import brooklyn.entity.basic.EntityInternal;
+import brooklyn.entity.rebind.RebindTestUtils;
+import brooklyn.entity.rebind.persister.ListeningObjectStore.RecordingTransactionListener;
+import brooklyn.management.ManagementContext;
+import brooklyn.test.entity.TestEntity;
+import brooklyn.util.text.Identifiers;
+import brooklyn.util.time.Duration;
+import brooklyn.util.time.Time;
+
+/** uses recorder to ensure not too much data is written */
+@Test
+public class BrooklynMementoPersisterInMemorySizeIntegrationTest extends BrooklynMementoPersisterTestFixture
{
+
+    protected RecordingTransactionListener recorder;
+    
+    protected ManagementContext newPersistingManagementContext() {
+        recorder = new RecordingTransactionListener("in-mem-test-"+Identifiers.makeRandomId(4));
+        return RebindTestUtils.managementContextBuilder(classLoader, 
+            new ListeningObjectStore(new InMemoryObjectStore(), recorder))
+            .persistPeriod(Duration.millis(100)).buildStarted();
+    }
+    
+    public void testPersistenceVolumeFast() throws IOException, TimeoutException, InterruptedException
{
+        doTestPersistenceVolume(50*1000, false);
+    }
+    @Test(groups="Integration")
+    public void testPersistenceVolumeWaiting() throws IOException, TimeoutException, InterruptedException
{
+        // by waiting we ensure there aren't extra writes going on
+        doTestPersistenceVolume(50*1000, true);
+    }
+    
+    protected void doTestPersistenceVolume(int bigBlockSize, boolean forceDelay) throws IOException,
TimeoutException, InterruptedException {
+        if (forceDelay) Time.sleep(Duration.FIVE_SECONDS);
+        else recorder.blockUntilDataWrittenExceeds(512, Duration.FIVE_SECONDS);
+        
+        long out1 = recorder.getBytesOut();
+        int filesOut1 = recorder.getCountDataOut();
+        Assert.assertTrue(out1>512, "should have written at least 0.5k, only wrote "+out1);
+        Assert.assertTrue(out1<20*1000, "should have written less than 20k, wrote "+out1);
+        Assert.assertTrue(filesOut1<20, "should have written fewer than 20 files, wrote
"+out1);
+        
+        ((EntityInternal)app).setAttribute(TestEntity.NAME, "hello world");
+        if (forceDelay) Time.sleep(Duration.FIVE_SECONDS);
+        else recorder.blockUntilDataWrittenExceeds(out1+10, Duration.FIVE_SECONDS);
+        
+        long out2 = recorder.getBytesOut();
+        Assert.assertTrue(out2-out1>10, "should have written more data");
+        int filesOut2 = recorder.getCountDataOut();
+        Assert.assertTrue(filesOut2>filesOut1, "should have written more files");
+        
+        Assert.assertTrue(out2<50*1000, "should have written less than 50k, wrote "+out1);
+        Assert.assertTrue(filesOut2<40, "should have written fewer than 40 files, wrote
"+out1);
+        
+        ((EntityInternal)entity).setAttribute(TestEntity.NAME, Identifiers.makeRandomId(bigBlockSize));
+        if (forceDelay) Time.sleep(Duration.FIVE_SECONDS);
+        else recorder.blockUntilDataWrittenExceeds(out2+bigBlockSize, Duration.FIVE_SECONDS);
+
+        long out3 = recorder.getBytesOut();
+        Assert.assertTrue(out3-out2 > bigBlockSize, "should have written 50k more data,
only wrote "+out3+" compared with "+out2);
+        int filesOut3 = recorder.getCountDataOut();
+        Assert.assertTrue(filesOut3>filesOut2, "should have written more files");
+        
+        Assert.assertTrue(out2<100*1000+bigBlockSize, "should have written less than 100k+block,
wrote "+out1);
+        Assert.assertTrue(filesOut2<60, "should have written fewer than 60 files, wrote
"+out1);
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java
b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java
index 04d0810..5ac0595 100644
--- a/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java
+++ b/core/src/test/java/brooklyn/entity/rebind/persister/BrooklynMementoPersisterInMemoryTest.java
@@ -1,24 +1,16 @@
 package brooklyn.entity.rebind.persister;
 
-import java.io.File;
-
 import org.testng.annotations.Test;
 
 import brooklyn.entity.rebind.RebindTestUtils;
 import brooklyn.management.ManagementContext;
 import brooklyn.util.time.Duration;
 
-/**
- * @author Andrea Turli
- */
 @Test
 public class BrooklynMementoPersisterInMemoryTest extends BrooklynMementoPersisterTestFixture
{
 
-    protected File mementoDir;
-    
     protected ManagementContext newPersistingManagementContext() {
         return RebindTestUtils.managementContextBuilder(classLoader, new InMemoryObjectStore())
             .persistPeriod(Duration.millis(10)).buildStarted();
     }
-    
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
b/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
new file mode 100644
index 0000000..82037de
--- /dev/null
+++ b/core/src/test/java/brooklyn/entity/rebind/persister/ListeningObjectStore.java
@@ -0,0 +1,185 @@
+package brooklyn.entity.rebind.persister;
+
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import brooklyn.management.ManagementContext;
+import brooklyn.util.text.Strings;
+import brooklyn.util.time.CountdownTimer;
+import brooklyn.util.time.Duration;
+
+import com.google.common.base.Preconditions;
+
+public class ListeningObjectStore implements PersistenceObjectStore {
+
+    protected final PersistenceObjectStore delegate;
+    protected final ObjectStoreTransactionListener listener;
+
+    public static interface ObjectStoreTransactionListener {
+        public void recordQueryOut(String summary, int size);
+        public void recordDataOut(String summary, int size);
+        public void recordDataIn(String summary, int size);
+    }
+    
+    public static class RecordingTransactionListener implements ObjectStoreTransactionListener
{
+        private static final Logger log = LoggerFactory.getLogger(ListeningObjectStore.RecordingTransactionListener.class);
+        
+        protected final String prefix;
+        protected final AtomicLong bytesIn = new AtomicLong();
+        protected final AtomicLong bytesOut = new AtomicLong();
+        protected final AtomicInteger countQueriesOut = new AtomicInteger();
+        protected final AtomicInteger countDataOut = new AtomicInteger();
+        protected final AtomicInteger countDataIn = new AtomicInteger();
+        
+        public RecordingTransactionListener(String prefix) {
+            this.prefix = prefix;
+        }
+        
+        public long getBytesIn() {
+            return bytesIn.get();
+        }
+        
+        public long getBytesOut() {
+            return bytesOut.get();
+        }
+        
+        public int getCountQueriesOut() {
+            return countQueriesOut.get();
+        }
+        
+        public int getCountDataOut() {
+            return countDataOut.get();
+        }
+        
+        public int getCountDataIn() {
+            return countDataIn.get();
+        }
+        
+        public String getTotalString() {
+            return "totals: out="+Strings.makeSizeString(bytesOut.get())+" in="+Strings.makeSizeString(bytesIn.get());
+        }
+        
+        @Override
+        public void recordQueryOut(String summary, int size) {
+            synchronized (this) { this.notifyAll(); }
+            bytesOut.addAndGet(size);
+            countQueriesOut.incrementAndGet();
+            log.info(prefix+" "+summary+" -->"+size+"; "+getTotalString());
+        }
+        
+        @Override
+        public void recordDataOut(String summary, int size) {
+            synchronized (this) { this.notifyAll(); }
+            bytesOut.addAndGet(size);
+            countDataOut.incrementAndGet();
+            log.info(prefix+" "+summary+" -->"+size+"; "+getTotalString());
+        }
+        
+        @Override
+        public void recordDataIn(String summary, int size) {
+            synchronized (this) { this.notifyAll(); }
+            bytesIn.addAndGet(size);
+            countDataIn.incrementAndGet();
+            log.info(prefix+" "+summary+" <--"+size+"; "+getTotalString());
+        }
+
+        public void blockUntilDataWrittenExceeds(long count, Duration timeout) throws InterruptedException,
TimeoutException {
+            CountdownTimer timer = CountdownTimer.newInstanceStarted(timeout);
+            synchronized (this) {
+                while (bytesOut.get()<count) {
+                    if (timer.isExpired())
+                        throw new TimeoutException();
+                    timer.waitOnForExpiry(this);
+                }
+            }
+        }
+    }
+
+    public ListeningObjectStore(PersistenceObjectStore delegate, ObjectStoreTransactionListener
listener) {
+        this.delegate = Preconditions.checkNotNull(delegate);
+        this.listener = Preconditions.checkNotNull(listener);
+    }
+
+    public String getSummaryName() {
+        return delegate.getSummaryName();
+    }
+
+    public StoreObjectAccessor newAccessor(String path) {
+        return new ListeningAccessor(path, delegate.newAccessor(path));
+    }
+
+    public void createSubPath(String subPath) {
+        listener.recordQueryOut("creating path "+subPath, 1+subPath.length());
+        delegate.createSubPath(subPath);
+    }
+
+    public List<String> listContentsWithSubPath(String subPath) {
+        listener.recordQueryOut("requesting list "+subPath, 1+subPath.length());
+        List<String> result = delegate.listContentsWithSubPath(subPath);
+        listener.recordDataIn("receiving list "+subPath, result.toString().length());
+        return result;
+    }
+
+    public void backupContents(String sourceSubPath, String targetSubPathForBackups) {
+        listener.recordQueryOut("backing up "+sourceSubPath+" to "+targetSubPathForBackups,

+            1+sourceSubPath.length()+targetSubPathForBackups.length());
+        delegate.backupContents(sourceSubPath, targetSubPathForBackups);
+    }
+
+    public void close() {
+        delegate.close();
+    }
+
+    public void prepareForUse(ManagementContext managementContext, PersistMode persistMode)
{
+        delegate.prepareForUse(managementContext, persistMode);
+    }
+
+    public void deleteCompletely() {
+        listener.recordDataOut("deleting completely", 1);
+        delegate.deleteCompletely();
+    }
+
+    public class ListeningAccessor implements StoreObjectAccessor {
+
+        protected final String path;
+        protected final StoreObjectAccessor delegate;
+        
+        public ListeningAccessor(String path, StoreObjectAccessor delegate) {
+            this.path = path;
+            this.delegate = delegate;
+        }
+        public boolean exists() {
+            return delegate.exists();
+        }
+        public void writeAsync(String val) {
+            listener.recordDataOut("writing "+path, val.length());
+            delegate.writeAsync(val);
+        }
+        public void append(String s) {
+            listener.recordDataOut("appending "+path, s.length());
+            delegate.append(s);
+        }
+        public void deleteAsync() {
+            listener.recordQueryOut("deleting "+path, path.length());
+            delegate.deleteAsync();
+        }
+        public String read() {
+            listener.recordQueryOut("requesting "+path, path.length());
+            String result = delegate.read();
+            listener.recordDataIn("reading "+path, result.length());
+            return result;
+        }
+        public void waitForWriteCompleted(Duration timeout) throws InterruptedException,
TimeoutException {
+            delegate.waitForWriteCompleted(timeout);
+        }
+        
+        
+        
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/94495f16/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java
----------------------------------------------------------------------
diff --git a/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java
b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java
new file mode 100644
index 0000000..1107913
--- /dev/null
+++ b/locations/jclouds/src/test/java/brooklyn/entity/rebind/persister/jclouds/BlobStoreCleaner.java
@@ -0,0 +1,54 @@
+package brooklyn.entity.rebind.persister.jclouds;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import brooklyn.config.BrooklynProperties;
+import brooklyn.location.basic.LocationConfigKeys;
+import brooklyn.location.cloud.CloudLocationConfig;
+import brooklyn.location.jclouds.JcloudsLocation;
+import brooklyn.test.entity.LocalManagementContextForTests;
+
+/** Utility for cleaning up after test leaks. Most should not leak of course, but if they
do... */
+public class BlobStoreCleaner {
+
+    private static String locationSpec = BlobStoreTest.PERSIST_TO_OBJECT_STORE_FOR_TEST_SPEC;
+
+    private static final Logger log = LoggerFactory.getLogger(BlobStoreCleaner.class);
+    
+    public static void main(String[] args) {
+        LocalManagementContextForTests mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault());

+        JcloudsLocation location = (JcloudsLocation) mgmt.getLocationRegistry().resolve(locationSpec);
+            
+        String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY),
"identity must not be null");
+        String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL),
"credential must not be null");
+        String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER),
"provider must not be null");
+        String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);
+
+        BlobStoreContext context = ContextBuilder.newBuilder(provider)
+                .credentials(identity, credential)
+                .endpoint(endpoint)
+                .buildView(BlobStoreContext.class);
+        
+        PageSet<? extends StorageMetadata> containers = context.getBlobStore().list();
+        for (StorageMetadata container: containers) {
+            if (container.getName().matches("brooklyn.*-test.*")
+                // to kill all containers here
+//                || container.getName().matches(".*")
+                ) {
+                log.info("killing - "+container.getName());
+                context.getBlobStore().deleteContainer(container.getName());
+            } else {
+                log.info("KEEPING - "+container.getName());
+            }
+        }
+        context.close();
+    }
+
+}


Mime
View raw message