incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [3/3] git commit: Creating a caching decompressor, helps row fetching performance.
Date Mon, 09 Dec 2013 03:02:32 GMT
Creating a caching decompressor, helps row fetching performance.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/985c72bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/985c72bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/985c72bf

Branch: refs/heads/master
Commit: 985c72bf79c3c906d468504f4796e3a9a584619b
Parents: 346a201
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Sun Dec 8 17:01:05 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Sun Dec 8 22:02:14 2013 -0500

----------------------------------------------------------------------
 .../apache/blur/manager/IndexManagerTest.java   |  2 +-
 .../apache/blur/lucene/codec/Blur021Codec.java  |  4 +-
 .../lucene/codec/Blur021StoredFieldsFormat.java | 25 ++++++
 .../lucene/codec/CachingCompressionMode.java    | 95 ++++++++++++++++++++
 4 files changed, 122 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
index 851df4f..5390a2c 100644
--- a/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/manager/IndexManagerTest.java
@@ -313,7 +313,7 @@ public class IndexManagerTest {
     }
 
     Trace.setStorage(oldReporter);
-
+    
   }
 
   private RowMutation getLargeRow(String rowId) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java
index 2d47705..311f40b 100644
--- a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021Codec.java
@@ -26,7 +26,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
 import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
-import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
 import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat;
 import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
 import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat;
@@ -34,7 +33,7 @@ import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 
 public class Blur021Codec extends Codec {
-  private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
+  private final StoredFieldsFormat fieldsFormat = new Blur021StoredFieldsFormat();
   private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
   private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat();
   private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat();
@@ -47,7 +46,6 @@ public class Blur021Codec extends Codec {
     }
   };
   
-  
   private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
     @Override
     public DocValuesFormat getDocValuesFormatForField(String field) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java
b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java
new file mode 100644
index 0000000..d2f678e
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/Blur021StoredFieldsFormat.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.codec;
+
+import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat;
+
+public final class Blur021StoredFieldsFormat extends CompressingStoredFieldsFormat {
+  public Blur021StoredFieldsFormat() {
+    super("Blur021StoredFields", CachingCompressionMode.CACHING_FAST, 1 << 14);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/985c72bf/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java
b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java
new file mode 100644
index 0000000..94f1304
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/codec/CachingCompressionMode.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.codec;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.codecs.compressing.Compressor;
+import org.apache.lucene.codecs.compressing.Decompressor;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
+
+public class CachingCompressionMode {
+
+  public static final CompressionMode CACHING_FAST = new CompressionMode() {
+
+    @Override
+    public Compressor newCompressor() {
+      return CompressionMode.FAST.newCompressor();
+    }
+
+    @Override
+    public Decompressor newDecompressor() {
+      return new CachedDecompressor(CompressionMode.FAST.newDecompressor());
+    }
+
+    @Override
+    public String toString() {
+      return "CACHING_FAST";
+    }
+
+  };
+
+  public static class CachedDecompressor extends Decompressor {
+
+    static class Entry {
+      IndexInput _indexInput;
+      long _position = -1;
+      BytesRef _bytesRef = new BytesRef();
+    }
+
+    private final Decompressor _decompressor;
+    private final ThreadLocal<Entry> _cache = new ThreadLocal<Entry>() {
+      @Override
+      protected Entry initialValue() {
+        return new Entry();
+      }
+    };
+
+    public CachedDecompressor(Decompressor decompressor) {
+      _decompressor = decompressor;
+    }
+
+    @Override
+    public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef
bytes) throws IOException {
+      if (in instanceof IndexInput) {
+        IndexInput indexInput = (IndexInput) in;
+        Entry entry = _cache.get();
+        long filePointer = indexInput.getFilePointer();
+        BytesRef cachedRef = entry._bytesRef;
+        if (entry._indexInput != indexInput || entry._position != filePointer) {
+          cachedRef.grow(originalLength);
+          _decompressor.decompress(in, originalLength, 0, originalLength, cachedRef);
+          entry._indexInput = indexInput;
+          entry._position = filePointer;
+        }
+        bytes.copyBytes(cachedRef);
+        bytes.offset = offset;
+        bytes.length = length;
+      } else {
+        _decompressor.decompress(in, originalLength, offset, length, bytes);
+      }
+    }
+
+    @Override
+    public Decompressor clone() {
+      return this;
+    }
+  }
+}


Mime
View raw message