lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dwe...@apache.org
Subject [lucene-solr] branch branch_8x updated: LUCENE-8642, LUCENE-8641: correct RamUsageTester.sizeOf's handling of ByteBuffers. Throw exceptions on denied reflection to catch problems early. This affects tests only.
Date Thu, 17 Jan 2019 11:25:40 GMT
This is an automated email from the ASF dual-hosted git repository.

dweiss pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/branch_8x by this push:
     new b4524d9  LUCENE-8642, LUCENE-8641: correct RamUsageTester.sizeOf's handling of ByteBuffers.
Throw exceptions on denied reflection to catch problems early. This affects tests only.
b4524d9 is described below

commit b4524d902df8536887b0caa108634fee8599b65c
Author: Dawid Weiss <dweiss@apache.org>
AuthorDate: Thu Jan 17 12:23:30 2019 +0100

    LUCENE-8642, LUCENE-8641: correct RamUsageTester.sizeOf's handling of ByteBuffers. Throw
exceptions on denied reflection to catch problems early. This affects tests only.
---
 .../apache/lucene/search/TermInSetQueryTest.java   |  1 -
 .../org/apache/lucene/util/RamUsageTester.java     | 46 ++++++++++++----------
 2 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
index 785b6d6..9633ea4 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
@@ -173,7 +173,6 @@ public class TermInSetQueryTest extends LuceneTestCase {
     QueryUtils.checkEqual(query1, query2);
   }
 
-  @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8641")
   public void testRamBytesUsed() {
     List<BytesRef> terms = new ArrayList<>();
     final int numTerms = 1000 + random().nextInt(1000);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
index 6437d8e..6a15007 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
@@ -31,6 +31,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.ToLongFunction;
@@ -146,34 +147,41 @@ public final class RamUsageTester {
          * and accumulate this object's shallow size. 
          */
         try {
-          ClassCache cachedInfo = classCache.get(obClazz);
-          if (cachedInfo == null) {
-            classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
-          }
-          
           boolean needsReflection = true;
           if (Constants.JRE_IS_MINIMUM_JAVA9 && obClazz.getName().startsWith("java."))
{
+            long alignedShallowInstanceSize = RamUsageEstimator.shallowSizeOf(ob);
+
             // Java 9: Best guess for some known types, as we cannot precisely look into
runtime classes:
             final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
             if (func != null) { // some simple type like String where the size is easy to
get from public properties
-              totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize
+ func.applyAsLong(ob), 
+              totalSize += accumulator.accumulateObject(ob, alignedShallowInstanceSize +
func.applyAsLong(ob),
                   Collections.emptyMap(), stack);
               needsReflection = false;
-            } else if (ob instanceof Iterable) {
-              final List<Object> values = StreamSupport.stream(((Iterable<?>)
ob).spliterator(), false)
-                  .collect(Collectors.toList());
-              totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+            } else if (ob instanceof ByteBuffer) {
+              // Approximate ByteBuffers with their underlying storage (ignores field overhead).
+              totalSize += byteArraySize(((ByteBuffer) ob).capacity());
               needsReflection = false;
             }  else if (ob instanceof Map) {
               final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
                   .flatMap(e -> Stream.of(e.getKey(), e.getValue()))
                   .collect(Collectors.toList());
-              totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+              totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER,
values, stack);
               totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
               needsReflection = false;
+            } else if (ob instanceof Iterable) {
+            final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(),
false)
+                .collect(Collectors.toList());
+              totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER,
values, stack);
+              needsReflection = false;
             }
           }
+
           if (needsReflection) {
+            ClassCache cachedInfo = classCache.get(obClazz);
+            if (cachedInfo == null) {
+              classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
+            }
+
             final Map<Field, Object> fieldValues = new HashMap<>();
             for (Field f : cachedInfo.referenceFields) {
               fieldValues.put(f, f.get(ob));
@@ -224,10 +232,6 @@ public final class RamUsageTester {
     private long charArraySize(int len) {
       return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
+ (long)Character.BYTES * len);
     }
-    
-    private long byteArraySize(int len) {
-      return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
+ len);
-    }
   });
   
   /**
@@ -268,11 +272,10 @@ public final class RamUsageTester {
                 f.setAccessible(true);
                 referenceFields.add(f);
               } catch (RuntimeException re) {
-                if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName()))
{
-                  // LUCENE-7595: this is Java 9, which prevents access to fields in foreign
modules
-                } else {
-                  throw re;
-                }
+                throw new RuntimeException(String.format(Locale.ROOT,
+                    "Can't access field %s of class %s for RAM estimation.",
+                    f.toString(),
+                    c.getName()), re);
               }
             }
           }
@@ -286,4 +289,7 @@ public final class RamUsageTester {
     });
   }
 
+  private static long byteArraySize(int len) {
+    return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
+ len);
+  }
 }


Mime
View raw message