lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jpou...@apache.org
Subject [2/2] lucene-solr:branch_6x: LUCENE-7647: CompressingStoredFieldsFormat should reclaim memory more aggressively.
Date Wed, 25 Jan 2017 16:06:19 GMT
LUCENE-7647: CompressingStoredFieldsFormat should reclaim memory more aggressively.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/03448807
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/03448807
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/03448807

Branch: refs/heads/branch_6x
Commit: 03448807a1b14657bdb8eb568f84df3d6ef09e01
Parents: 271dc1c
Author: Adrien Grand <jpountz@gmail.com>
Authored: Wed Jan 25 16:15:04 2017 +0100
Committer: Adrien Grand <jpountz@gmail.com>
Committed: Wed Jan 25 16:39:56 2017 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  4 ++
 .../CompressingStoredFieldsWriter.java          |  5 +-
 .../codecs/compressing/CompressionMode.java     | 49 ++++++++++++++------
 .../lucene/codecs/compressing/Compressor.java   |  3 +-
 .../dummy/DummyCompressingCodec.java            |  3 ++
 5 files changed, 46 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/03448807/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9596541..fcde6a2 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -67,6 +67,10 @@ Bug Fixes
 * LUCENE-7657: Fixed potential memory leak in the case that a (Span)TermQuery
   with a TermContext is cached. (Adrien Grand)
 
+* LUCENE-7647: Made stored fields reclaim native memory more aggressively when
+  configured with BEST_COMPRESSION. This could otherwise result in out-of-memory
+  issues. (Adrien Grand)
+
 ======================= Lucene 6.4.0 =======================
 
 API Changes

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/03448807/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
index 7f6c74e..8bee77d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
@@ -81,7 +81,7 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter
{
   private CompressingStoredFieldsIndexWriter indexWriter;
   private IndexOutput fieldsStream;
 
-  private final Compressor compressor;
+  private Compressor compressor;
   private final CompressionMode compressionMode;
   private final int chunkSize;
   private final int maxDocsPerChunk;
@@ -141,10 +141,11 @@ public final class CompressingStoredFieldsWriter extends StoredFieldsWriter
{
   @Override
   public void close() throws IOException {
     try {
-      IOUtils.close(fieldsStream, indexWriter);
+      IOUtils.close(fieldsStream, indexWriter, compressor);
     } finally {
       fieldsStream = null;
       indexWriter = null;
+      compressor = null;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/03448807/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
index 326eba3..53a84cb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java
@@ -164,6 +164,10 @@ public abstract class CompressionMode {
       LZ4.compress(bytes, off, len, out, ht);
     }
 
+    @Override
+    public void close() throws IOException {
+      // no-op
+    }
   }
 
   private static final class LZ4HighCompressor extends Compressor {
@@ -180,15 +184,17 @@ public abstract class CompressionMode {
       LZ4.compressHC(bytes, off, len, out, ht);
     }
 
+    @Override
+    public void close() throws IOException {
+      // no-op
+    }
   }
 
   private static final class DeflateDecompressor extends Decompressor {
 
-    final Inflater decompressor;
     byte[] compressed;
 
     DeflateDecompressor() {
-      decompressor = new Inflater(true);
       compressed = new byte[0];
     }
 
@@ -207,20 +213,24 @@ public abstract class CompressionMode {
       in.readBytes(compressed, 0, compressedLength);
       compressed[compressedLength] = 0; // explicitly set dummy byte to 0
 
-      decompressor.reset();
-      // extra "dummy byte"
-      decompressor.setInput(compressed, 0, paddedLength);
-
-      bytes.offset = bytes.length = 0;
-      bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+      final Inflater decompressor = new Inflater(true);
       try {
-        bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
-      } catch (DataFormatException e) {
-        throw new IOException(e);
-      }
-      if (!decompressor.finished()) {
-        throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()

-                                                            + ", needsDict=" + decompressor.needsDictionary(),
in);
+        // extra "dummy byte"
+        decompressor.setInput(compressed, 0, paddedLength);
+
+        bytes.offset = bytes.length = 0;
+        bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength);
+        try {
+          bytes.length = decompressor.inflate(bytes.bytes, bytes.length, originalLength);
+        } catch (DataFormatException e) {
+          throw new IOException(e);
+        }
+        if (!decompressor.finished()) {
+          throw new CorruptIndexException("Invalid decoder state: needsInput=" + decompressor.needsInput()
+                                                              + ", needsDict=" + decompressor.needsDictionary(),
in);
+        }
+      } finally {
+        decompressor.end();
       }
       if (bytes.length != originalLength) {
         throw new CorruptIndexException("Lengths mismatch: " + bytes.length + " != " + originalLength,
in);
@@ -240,6 +250,7 @@ public abstract class CompressionMode {
 
     final Deflater compressor;
     byte[] compressed;
+    boolean closed;
 
     DeflateCompressor(int level) {
       compressor = new Deflater(level, true);
@@ -275,6 +286,14 @@ public abstract class CompressionMode {
       out.writeBytes(compressed, totalCount);
     }
 
+    @Override
+    public void close() throws IOException {
+      if (closed == false) {
+        compressor.end();
+        closed = true;
+      }
+    }
+
   }
 
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/03448807/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
index bd2fadb..f95246c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Compressor.java
@@ -17,6 +17,7 @@
 package org.apache.lucene.codecs.compressing;
 
 
+import java.io.Closeable;
 import java.io.IOException;
 
 import org.apache.lucene.store.DataOutput;
@@ -24,7 +25,7 @@ import org.apache.lucene.store.DataOutput;
 /**
  * A data compressor.
  */
-public abstract class Compressor {
+public abstract class Compressor implements Closeable {
 
   /** Sole constructor, typically called from sub-classes. */
   protected Compressor() {}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/03448807/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
index d15adad..167418e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/dummy/DummyCompressingCodec.java
@@ -79,6 +79,9 @@ public class DummyCompressingCodec extends CompressingCodec {
       out.writeBytes(bytes, off, len);
     }
 
+    @Override
+    public void close() throws IOException {};
+
   };
 
   /** Constructor that allows to configure the chunk size. */


Mime
View raw message