geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tush...@apache.org
Subject [69/79] incubator-geode git commit: GEODE-306: support printing of off-heap compressed nodes
Date Tue, 03 Nov 2015 11:26:31 GMT
GEODE-306: support printing of off-heap compressed nodes

Before when a chunk of off-heap compressed memory tried to
get the type of the data stored in the chunk it needed to
decompress the data. Since it did not know what decompressor
to use it threw an UnsupportedOperationException.
Now for a compressed chunk it returns the DataType as either
a "compressed object of size xxx" or "compressed byte[xxx]".
It does not do a decompression.
Unit test coverage was added.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/8ff01baa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/8ff01baa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/8ff01baa

Branch: refs/heads/feature/GEODE-17
Commit: 8ff01baa50917114c4fdc105ef43527f7e975f19
Parents: 208e9a1
Author: Darrel Schneider <dschneider@pivotal.io>
Authored: Tue Sep 1 16:40:43 2015 -0700
Committer: tushark <tushark@apache.org>
Committed: Tue Nov 3 16:52:44 2015 +0530

----------------------------------------------------------------------
 .../offheap/SimpleMemoryAllocatorImpl.java      |  8 +++-
 .../offheap/OffHeapValidationJUnitTest.java     | 45 +++++++++++++++-----
 2 files changed, 41 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8ff01baa/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
index 29319e0..7cf1656 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java
@@ -3341,7 +3341,13 @@ public final class SimpleMemoryAllocatorImpl implements MemoryAllocator,
MemoryI
       }
       if (!isSerialized()) {
         // byte array
-        return "byte[" + ((Chunk)this.block).getDataSize() + "]";
+        if (isCompressed()) {
+          return "compressed byte[" + ((Chunk)this.block).getDataSize() + "]";
+        } else {
+          return "byte[" + ((Chunk)this.block).getDataSize() + "]";
+        }
+      } else if (isCompressed()) {
+        return "compressed object of size " + ((Chunk)this.block).getDataSize();
       }
       //Object obj = EntryEventImpl.deserialize(((Chunk)this.block).getRawBytes());
       byte[] bytes = ((Chunk)this.block).getRawBytes();

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/8ff01baa/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
index 7991872..1b40f2a 100755
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java
@@ -5,10 +5,13 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.sql.Timestamp;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -33,9 +36,13 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.gemstone.gemfire.DataSerializer;
 import com.gemstone.gemfire.cache.CacheFactory;
 import com.gemstone.gemfire.cache.Region;
 import com.gemstone.gemfire.cache.RegionShortcut;
+import com.gemstone.gemfire.compression.SnappyCompressor;
+import com.gemstone.gemfire.internal.HeapDataOutputStream;
+import com.gemstone.gemfire.internal.Version;
 import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
 import com.gemstone.gemfire.internal.cache.LocalRegion;
 import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl.Chunk;
@@ -87,7 +94,7 @@ public class OffHeapValidationJUnitTest {
   }
   
   @Test
-  public void testMemoryInspection() {
+  public void testMemoryInspection() throws IOException {
     // validate initial state
     MemoryAllocator allocator = this.cache.getOffHeapStore();
     assertNotNull(allocator);
@@ -113,6 +120,7 @@ public class OffHeapValidationJUnitTest {
     
     // create off-heap region
     Region<Object, Object> region = this.cache.createRegionFactory(getRegionShortcut()).setOffHeap(true).create(getRegionName());
+    Region<Object, Object> compressedRegion = this.cache.createRegionFactory(getRegionShortcut()).setOffHeap(true).setCompressor(SnappyCompressor.getDefaultInstance()).create(getRegionName()+"Compressed");
     
     // perform some ops
     List<ExpectedValues> expected = new ArrayList<ExpectedValues>();
@@ -120,8 +128,10 @@ public class OffHeapValidationJUnitTest {
     // Chunk.OFF_HEAP_HEADER_SIZE + 4 ?
     
     putString(region, expected);
+    putCompressedString(compressedRegion, expected);
     putDate(region, expected);
     putByteArray(region, expected);
+    putCompressedByteArray(compressedRegion, expected);
     putByteArrayArray(region, expected);
     putShortArray(region, expected);
     putStringArray(region, expected);
@@ -189,8 +199,11 @@ public class OffHeapValidationJUnitTest {
         assertTrue(obj instanceof String);
         assertEquals("this is a string", (String)obj);
       }
-      if (values.dataType.contains("[")) { //for (int j = 0; j < ((byte[])values.dataValue).length;
j++) {
-        // TODO
+      if ((values.dataType.contains("byte [") && values.dataType.lastIndexOf('[')
== values.dataType.indexOf('[')) || values.dataType.startsWith("compressed")) {
+        assertTrue("for dataType=" + values.dataType + " expected " + Arrays.toString((byte[])values.dataValue)
+ " but was " + Arrays.toString((byte[])block.getDataValue()),
+            Arrays.equals((byte[])values.dataValue, (byte[])block.getDataValue()));
+      } else if (values.dataType.contains("[")) {
+        // TODO: multiple dimension arrays or non-byte arrays
       } else if (values.dataValue instanceof Collection) {
         int diff = joint((Collection<?>)values.dataValue, (Collection<?>)block.getDataValue());
         assertEquals(i + ":" + values.dataType, 0, diff);
@@ -216,14 +229,6 @@ public class OffHeapValidationJUnitTest {
     
   }
   
-  @Test
-  public void testCompaction() {
-    // create fragmented state
-    // validate fragmented
-    // perform compaction
-    // validate freed fragments
-  }
-  
   /**
    * Returns -1 if c1 is missing an element in c2, 1 if c2 is missing an element
    * in c1, or 0 is they contain the exact same elements.
@@ -289,6 +294,17 @@ public class OffHeapValidationJUnitTest {
     expected.add(new ExpectedValues(value, value.length()*2, "java.lang.String", -1, getMemoryAddress(region,
key), 1, 0, false, true));
   }
   
+  private void putCompressedString(Region<Object, Object> region, List<ExpectedValues>
expected) throws IOException {
+    String key = "keyString";
+    String value = "this is a string";
+    region.put(key, value);
+    HeapDataOutputStream hdos = new HeapDataOutputStream(Version.CURRENT);
+    DataSerializer.writeObject(value, hdos);
+    byte[] uncompressedBytes = hdos.toByteArray();
+    byte[] expectedValue = SnappyCompressor.getDefaultInstance().compress(uncompressedBytes);
+    expected.add(new ExpectedValues(expectedValue, 32, "compressed object of size " + expectedValue.length,
-1, getMemoryAddress(region, key), 1, 0, true, true));
+  }
+
   private void putDate(Region<Object, Object> region, List<ExpectedValues> expected)
{
     String key = "keyDate";
     Date value = new Date();
@@ -302,6 +318,13 @@ public class OffHeapValidationJUnitTest {
     region.put(key, value);
     expected.add(new ExpectedValues(value, 24, "byte[10]", -1, getMemoryAddress(region, key),
1, 0, false, false));
   }
+  private void putCompressedByteArray(Region<Object, Object> region, List<ExpectedValues>
expected) throws IOException {
+    String key = "keyByteArray";
+    byte[] value = new byte[10];
+    region.put(key, value);
+    byte[] expectedValue = SnappyCompressor.getDefaultInstance().compress(value);
+    expected.add(new ExpectedValues(expectedValue, 24, "compressed byte[" + expectedValue.length
+ "]", -1, getMemoryAddress(region, key), 1, 0, true, false));
+  }
   
   private void putByteArrayArray(Region<Object, Object> region, List<ExpectedValues>
expected) {
     String key = "keyByteArrayArray";


Mime
View raw message