avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [11/43] avro git commit: AVRO-1821: Fix possible memory leak of Schemas in ReflectData. Contributed by Byran Harclerode.
Date Sat, 14 May 2016 23:43:44 GMT
AVRO-1821: Fix possible memory leak of Schemas in ReflectData. Contributed by Byran Harclerode.


Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/ec8a0918
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/ec8a0918
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/ec8a0918

Branch: refs/heads/branch-1.8
Commit: ec8a091819a25bccf03adc868449f57f9c076d19
Parents: 6045065
Author: Ryan Blue <blue@apache.org>
Authored: Sat Apr 16 14:10:45 2016 -0700
Committer: Ryan Blue <blue@apache.org>
Committed: Sat May 14 16:43:06 2016 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 ++
 lang/java/avro/pom.xml                          |  5 +++
 .../org/apache/avro/reflect/ReflectData.java    | 10 +++---
 .../apache/avro/reflect/TestReflectData.java    | 35 ++++++++++++++++++++
 4 files changed, 49 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/avro/blob/ec8a0918/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dd41810..f656de3 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -31,6 +31,9 @@ Trunk (not yet released)
 
     AVRO-1829. C++ documentation improvements (William S Fulton via thiru)
 
+    AVRO-1821: Java: Fix possible memory leak in ReflectData accessor cache.
+    (Bryan Harclerode via blue)
+
 Avro 1.8.0 (22 January 2016)
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/avro/blob/ec8a0918/lang/java/avro/pom.xml
----------------------------------------------------------------------
diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml
index 48a3d42..bc7b191 100644
--- a/lang/java/avro/pom.xml
+++ b/lang/java/avro/pom.xml
@@ -198,6 +198,11 @@
       <artifactId>joda-time</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-all</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/avro/blob/ec8a0918/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java b/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
index 827e1fa..9e13e7c 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
@@ -38,6 +38,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.MapMaker;
 import org.apache.avro.AvroRemoteException;
 import org.apache.avro.AvroRuntimeException;
 import org.apache.avro.AvroTypeException;
@@ -52,6 +54,7 @@ import org.apache.avro.generic.GenericFixed;
 import org.apache.avro.generic.IndexedRecord;
 import org.apache.avro.io.BinaryData;
 import org.apache.avro.util.ClassUtils;
+import org.apache.avro.util.WeakIdentityHashMap;
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.io.DatumWriter;
 import org.apache.avro.specific.FixedSize;
@@ -230,13 +233,12 @@ public class ReflectData extends SpecificData {
   static final ConcurrentHashMap<Class<?>, ClassAccessorData> 
     ACCESSOR_CACHE = new ConcurrentHashMap<Class<?>, ClassAccessorData>();
 
-  private static class ClassAccessorData {
+  static class ClassAccessorData {
     private final Class<?> clazz;
     private final Map<String, FieldAccessor> byName =
         new HashMap<String, FieldAccessor>();
-    private final IdentityHashMap<Schema, FieldAccessor[]> bySchema =
-        new IdentityHashMap<Schema, FieldAccessor[]>();
-        
+    final Map<Schema, FieldAccessor[]> bySchema = new MapMaker().weakKeys().makeMap();
+
     private ClassAccessorData(Class<?> c) {
       clazz = c;
       for(Field f : getFields(c, false)) {

http://git-wip-us.apache.org/repos/asf/avro/blob/ec8a0918/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectData.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectData.java b/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectData.java
new file mode 100644
index 0000000..46645c2
--- /dev/null
+++ b/lang/java/avro/src/test/java/org/apache/avro/reflect/TestReflectData.java
@@ -0,0 +1,35 @@
+package org.apache.avro.reflect;
+
+import org.apache.avro.Schema;
+import org.junit.Test;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertThat;
+
+public class TestReflectData {
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testWeakSchemaCaching() throws Exception {
+    int numSchemas = 1000000;
+    for (int i = 0; i < numSchemas; i++) {
+      // Create schema
+      Schema schema = Schema.createRecord("schema", null, null, false);
+      schema.setFields(Collections.<Schema.Field>emptyList());
+
+      ReflectData.get().getRecordState(new Object(), schema);
+    }
+
+    // Reflect the number of schemas currently in the cache
+    ReflectData.ClassAccessorData classData = ReflectData.ACCESSOR_CACHE
+        .get(Object.class);
+
+    System.gc(); // Not guaranteed, but seems to be reliable enough
+
+    assertThat("ReflectData cache should release references",
+        classData.bySchema.size(), lessThan(numSchemas));
+  }
+}


Mime
View raw message