incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/2] git commit: Adding a fix/hack to be able to externalize the FST data arrays in Lucene.
Date Fri, 21 Feb 2014 02:40:34 GMT
Adding a fix/hack to be able to externalize the FST data arrays in Lucene.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/393c9f53
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/393c9f53
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/393c9f53

Branch: refs/heads/apache-blur-0.2
Commit: 393c9f53d2b6e0e004a2813e3cc7ad94fd4f01bc
Parents: d0e26ca
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Feb 20 21:39:49 2014 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Feb 20 21:39:49 2014 -0500

----------------------------------------------------------------------
 .../org/apache/blur/lucene/fst/ByteArray.java   |  42 ++
 .../blur/lucene/fst/ByteArrayFactory.java       |  70 +++
 .../blur/lucene/fst/ByteArrayPrimitive.java     |  72 +++
 .../lucene/fst/ByteArrayPrimitiveFactory.java   |  32 ++
 .../org/apache/lucene/util/fst/BytesStore.java  | 561 +++++++++++++++++++
 .../org/apache/blur/utils/BlurConstants.java    |   1 +
 .../src/main/resources/blur-default.properties  |   3 +
 7 files changed, 781 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java
new file mode 100644
index 0000000..5d28e8c
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArray.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.fst;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+public abstract class ByteArray {
+
+  public abstract int length();
+
+  public abstract void put(int position, byte b);
+
+  public abstract byte get(int position);
+
+  public abstract void put(int position, byte[] b, int offset, int len);
+
+  public abstract void get(int position, byte[] b, int offset, int len);
+
+  public abstract void readBytes(DataInput in, int offset, int length) throws IOException;
+
+  public abstract void writeBytes(DataOutput out, int offset, int length) throws IOException;
+
+  public abstract void copy(int position, ByteArray dest, int destOffset, int len);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java
new file mode 100644
index 0000000..9bdb745
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayFactory.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.fst;
+
+import static org.apache.blur.utils.BlurConstants.BLUR_LUCENE_FST_BYTEARRAY_FACTORY;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+
+import org.apache.blur.BlurConfiguration;
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+
+public abstract class ByteArrayFactory {
+
+  private static Log LOG = LogFactory.getLog(ByteArrayFactory.class);
+
+  private final static ByteArrayFactory _factory;
+
+  static {
+    BlurConfiguration configuration;
+    try {
+      configuration = new BlurConfiguration();
+    } catch (IOException e) {
+      LOG.error("Error while trying to open blur config.", e);
+      try {
+        configuration = new BlurConfiguration(false);
+      } catch (IOException ex) {
+        throw new RuntimeException();
+      }
+    }
+    ByteArrayFactory factory = new ByteArrayPrimitiveFactory(configuration);
+    try {
+      String className = configuration.get(BLUR_LUCENE_FST_BYTEARRAY_FACTORY);
+      if (!(className == null || className.isEmpty())) {
+        Class<?> clazz = Class.forName(className);
+        Constructor<?> constructor = clazz.getConstructor(new Class[] { BlurConfiguration.class
});
+        factory = (ByteArrayFactory) constructor.newInstance(new Object[] { constructor });
+      }
+    } catch (Exception e) {
+      LOG.error("Error while trying create new bytearray factory for lucene bytestore.",
e);
+    }
+    _factory = factory;
+  }
+
+  public ByteArrayFactory(BlurConfiguration configuration) {
+
+  }
+
+  public abstract ByteArray newByteArray(int size);
+
+  public static ByteArrayFactory getDefaultFactory() {
+    return _factory;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java
new file mode 100644
index 0000000..3c03879
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitive.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.fst;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+public class ByteArrayPrimitive extends ByteArray {
+
+  private final byte[] _bytes;
+
+  public ByteArrayPrimitive(int size) {
+    _bytes = new byte[size];
+  }
+
+  @Override
+  public int length() {
+    return _bytes.length;
+  }
+
+  @Override
+  public void put(int position, byte b) {
+    _bytes[position] = b;
+  }
+
+  @Override
+  public byte get(int position) {
+    return _bytes[position];
+  }
+
+  @Override
+  public void put(int position, byte[] b, int offset, int len) {
+    System.arraycopy(b, offset, _bytes, position, len);
+  }
+
+  @Override
+  public void get(int position, byte[] b, int offset, int len) {
+    System.arraycopy(_bytes, position, b, offset, len);
+  }
+
+  @Override
+  public void readBytes(DataInput in, int offset, int length) throws IOException {
+    in.readBytes(_bytes, offset, length);
+  }
+
+  @Override
+  public void writeBytes(DataOutput out, int offset, int length) throws IOException {
+    out.writeBytes(_bytes, offset, length);
+  }
+
+  @Override
+  public void copy(int position, ByteArray dest, int destOffset, int len) {
+    System.arraycopy(_bytes, position, ((ByteArrayPrimitive) dest)._bytes, destOffset, len);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java
b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java
new file mode 100644
index 0000000..e5f73a2
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/fst/ByteArrayPrimitiveFactory.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.lucene.fst;
+
+import org.apache.blur.BlurConfiguration;
+
+public class ByteArrayPrimitiveFactory extends ByteArrayFactory {
+
+  public ByteArrayPrimitiveFactory(BlurConfiguration configuration) {
+    super(configuration);
+  }
+
+  @Override
+  public ByteArray newByteArray(int size) {
+    return new ByteArrayPrimitive(size);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java b/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java
new file mode 100644
index 0000000..6bac91b
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/lucene/util/fst/BytesStore.java
@@ -0,0 +1,561 @@
+package org.apache.lucene.util.fst;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.blur.lucene.fst.ByteArray;
+import org.apache.blur.lucene.fst.ByteArrayFactory;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+
+// TODO: merge with PagedBytes, except PagedBytes doesn't
+// let you read while writing which FST needs
+
+class BytesStore extends DataOutput {
+
+  private final List<ByteArray> blocks = new ArrayList<ByteArray>();
+  private final static ByteArrayFactory factory = ByteArrayFactory.getDefaultFactory();
+
+  private final int blockSize;
+  private final int blockBits;
+  private final int blockMask;
+
+  private ByteArray current;
+  private int nextWrite;
+
+  public BytesStore(int blockBits) {
+    this.blockBits = blockBits;
+    blockSize = 1 << blockBits;
+    blockMask = blockSize - 1;
+    nextWrite = blockSize;
+  }
+
+  /** Pulls bytes from the provided IndexInput. */
+  public BytesStore(DataInput in, long numBytes, int maxBlockSize) throws IOException {
+    int blockSize = 2;
+    int blockBits = 1;
+    while (blockSize < numBytes && blockSize < maxBlockSize) {
+      blockSize *= 2;
+      blockBits++;
+    }
+    this.blockBits = blockBits;
+    this.blockSize = blockSize;
+    this.blockMask = blockSize - 1;
+    long left = numBytes;
+    while (left > 0) {
+      final int chunk = (int) Math.min(blockSize, left);
+      ByteArray block = factory.newByteArray(chunk);
+      block.readBytes(in, 0, block.length());
+      blocks.add(block);
+      left -= chunk;
+    }
+
+    // So .getPosition still works
+    nextWrite = blocks.get(blocks.size() - 1).length();
+  }
+
+  /**
+   * Absolute write byte; you must ensure dest is < max position written so far.
+   */
+  public void writeByte(int dest, byte b) {
+    int blockIndex = dest >> blockBits;
+    ByteArray block = blocks.get(blockIndex);
+    block.put(dest & blockMask, b);
+  }
+
+  @Override
+  public void writeByte(byte b) {
+    if (nextWrite == blockSize) {
+      current = factory.newByteArray(blockSize);
+      blocks.add(current);
+      nextWrite = 0;
+    }
+    current.put(nextWrite++, b);
+  }
+
+  @Override
+  public void writeBytes(byte[] b, int offset, int len) {
+    while (len > 0) {
+      int chunk = blockSize - nextWrite;
+      if (len <= chunk) {
+        arraycopy(b, offset, current, nextWrite, len);
+        nextWrite += len;
+        break;
+      } else {
+        if (chunk > 0) {
+          arraycopy(b, offset, current, nextWrite, chunk);
+          offset += chunk;
+          len -= chunk;
+        }
+        current = factory.newByteArray(blockSize);
+        blocks.add(current);
+        nextWrite = 0;
+      }
+    }
+  }
+
+  int getBlockBits() {
+    return blockBits;
+  }
+
+  void writeBytes(long dest, byte[] b, int offset, int len) {
+    ByteArray byteArray = factory.newByteArray(len);
+    byteArray.put(0, b, offset, len);
+    writeBytes(dest, byteArray, 0, len);
+  }
+
+  /**
+   * Absolute writeBytes without changing the current position. Note: this
+   * cannot "grow" the bytes, so you must only call it on already written parts.
+   */
+  void writeBytes(long dest, ByteArray b, int offset, int len) {
+    // System.out.println("  BS.writeBytes dest=" + dest + " offset=" + offset +
+    // " len=" + len);
+    assert dest + len <= getPosition() : "dest=" + dest + " pos=" + getPosition() + "
len=" + len;
+
+    // Note: weird: must go "backwards" because copyBytes
+    // calls us with overlapping src/dest. If we
+    // go forwards then we overwrite bytes before we can
+    // copy them:
+
+    /*
+     * int blockIndex = dest >> blockBits; int upto = dest & blockMask; byte[]
+     * block = blocks.get(blockIndex); while (len > 0) { int chunk = blockSize -
+     * upto; System.out.println("    cycle chunk=" + chunk + " len=" + len); if
+     * (len <= chunk) { System.arraycopy(b, offset, block, upto, len); break; }
+     * else { System.arraycopy(b, offset, block, upto, chunk); offset += chunk;
+     * len -= chunk; blockIndex++; block = blocks.get(blockIndex); upto = 0; } }
+     */
+
+    final long end = dest + len;
+    int blockIndex = (int) (end >> blockBits);
+    int downTo = (int) (end & blockMask);
+    if (downTo == 0) {
+      blockIndex--;
+      downTo = blockSize;
+    }
+    ByteArray block = blocks.get(blockIndex);
+
+    while (len > 0) {
+      // System.out.println("    cycle downTo=" + downTo + " len=" + len);
+      if (len <= downTo) {
+        // System.out.println("      final: offset=" + offset + " len=" + len +
+        // " dest=" + (downTo-len));
+        arraycopy(b, offset, block, downTo - len, len);
+        break;
+      } else {
+        len -= downTo;
+        // System.out.println("      partial: offset=" + (offset + len) +
+        // " len=" + downTo + " dest=0");
+        arraycopy(b, offset + len, block, 0, downTo);
+        blockIndex--;
+        block = blocks.get(blockIndex);
+        downTo = blockSize;
+      }
+    }
+  }
+
+  /**
+   * Absolute copy bytes self to self, without changing the position. Note: this
+   * cannot "grow" the bytes, so must only call it on already written parts.
+   */
+  public void copyBytes(long src, long dest, int len) {
+    // System.out.println("BS.copyBytes src=" + src + " dest=" + dest + " len="
+    // + len);
+    assert src < dest;
+
+    // Note: weird: must go "backwards" because copyBytes
+    // calls us with overlapping src/dest. If we
+    // go forwards then we overwrite bytes before we can
+    // copy them:
+
+    /*
+     * int blockIndex = src >> blockBits; int upto = src & blockMask; byte[]
+     * block = blocks.get(blockIndex); while (len > 0) { int chunk = blockSize -
+     * upto; System.out.println("  cycle: chunk=" + chunk + " len=" + len); if
+     * (len <= chunk) { writeBytes(dest, block, upto, len); break; } else {
+     * writeBytes(dest, block, upto, chunk); blockIndex++; block =
+     * blocks.get(blockIndex); upto = 0; len -= chunk; dest += chunk; } }
+     */
+
+    long end = src + len;
+
+    int blockIndex = (int) (end >> blockBits);
+    int downTo = (int) (end & blockMask);
+    if (downTo == 0) {
+      blockIndex--;
+      downTo = blockSize;
+    }
+    ByteArray block = blocks.get(blockIndex);
+
+    while (len > 0) {
+      // System.out.println("  cycle downTo=" + downTo);
+      if (len <= downTo) {
+        // System.out.println("    finish");
+        writeBytes(dest, block, downTo - len, len);
+        break;
+      } else {
+        // System.out.println("    partial");
+        len -= downTo;
+        writeBytes(dest + len, block, 0, downTo);
+        blockIndex--;
+        block = blocks.get(blockIndex);
+        downTo = blockSize;
+      }
+    }
+  }
+
+  /**
+   * Writes an int at the absolute position without changing the current
+   * pointer.
+   */
+  public void writeInt(long pos, int value) {
+    int blockIndex = (int) (pos >> blockBits);
+    int upto = (int) (pos & blockMask);
+    ByteArray block = blocks.get(blockIndex);
+    int shift = 24;
+    for (int i = 0; i < 4; i++) {
+      block.put(upto++, (byte) (value >> shift));
+      shift -= 8;
+      if (upto == blockSize) {
+        upto = 0;
+        blockIndex++;
+        block = blocks.get(blockIndex);
+      }
+    }
+  }
+
+  /** Reverse from srcPos, inclusive, to destPos, inclusive. */
+  public void reverse(long srcPos, long destPos) {
+    assert srcPos < destPos;
+    assert destPos < getPosition();
+    // System.out.println("reverse src=" + srcPos + " dest=" + destPos);
+
+    int srcBlockIndex = (int) (srcPos >> blockBits);
+    int src = (int) (srcPos & blockMask);
+    ByteArray srcBlock = blocks.get(srcBlockIndex);
+
+    int destBlockIndex = (int) (destPos >> blockBits);
+    int dest = (int) (destPos & blockMask);
+    ByteArray destBlock = blocks.get(destBlockIndex);
+    // System.out.println("  srcBlock=" + srcBlockIndex + " destBlock=" +
+    // destBlockIndex);
+
+    int limit = (int) (destPos - srcPos + 1) / 2;
+    for (int i = 0; i < limit; i++) {
+      // System.out.println("  cycle src=" + src + " dest=" + dest);
+      byte b = srcBlock.get(src);
+      srcBlock.put(src, destBlock.get(dest));
+      destBlock.put(dest, b);
+      src++;
+      if (src == blockSize) {
+        srcBlockIndex++;
+        srcBlock = blocks.get(srcBlockIndex);
+        // System.out.println("  set destBlock=" + destBlock + " srcBlock=" +
+        // srcBlock);
+        src = 0;
+      }
+
+      dest--;
+      if (dest == -1) {
+        destBlockIndex--;
+        destBlock = blocks.get(destBlockIndex);
+        // System.out.println("  set destBlock=" + destBlock + " srcBlock=" +
+        // srcBlock);
+        dest = blockSize - 1;
+      }
+    }
+  }
+
+  public void skipBytes(int len) {
+    while (len > 0) {
+      int chunk = blockSize - nextWrite;
+      if (len <= chunk) {
+        nextWrite += len;
+        break;
+      } else {
+        len -= chunk;
+        current = factory.newByteArray(blockSize);
+        blocks.add(current);
+        nextWrite = 0;
+      }
+    }
+  }
+
+  public long getPosition() {
+    return ((long) blocks.size() - 1) * blockSize + nextWrite;
+  }
+
+  /**
+   * Pos must be less than the max position written so far! Ie, you cannot
+   * "grow" the file with this!
+   */
+  public void truncate(long newLen) {
+    assert newLen <= getPosition();
+    assert newLen >= 0;
+    int blockIndex = (int) (newLen >> blockBits);
+    nextWrite = (int) (newLen & blockMask);
+    if (nextWrite == 0) {
+      blockIndex--;
+      nextWrite = blockSize;
+    }
+    blocks.subList(blockIndex + 1, blocks.size()).clear();
+    if (newLen == 0) {
+      current = null;
+    } else {
+      current = blocks.get(blockIndex);
+    }
+    assert newLen == getPosition();
+  }
+
+  public void finish() {
+    if (current != null) {
+      ByteArray lastBuffer = factory.newByteArray(nextWrite);
+      arraycopy(current, 0, lastBuffer, 0, nextWrite);
+      blocks.set(blocks.size() - 1, lastBuffer);
+      current = null;
+    }
+  }
+
+  /** Writes all of our bytes to the target {@link DataOutput}. */
+  public void writeTo(DataOutput out) throws IOException {
+    for (ByteArray block : blocks) {
+      block.writeBytes(out, 0, block.length());
+    }
+  }
+
+  public FST.BytesReader getForwardReader() {
+    if (blocks.size() == 1) {
+      return new ForwardBytesReaderByteArray(blocks.get(0));
+    }
+    return new FST.BytesReader() {
+      private ByteArray current;
+      private int nextBuffer;
+      private int nextRead = blockSize;
+
+      @Override
+      public byte readByte() {
+        if (nextRead == blockSize) {
+          current = blocks.get(nextBuffer++);
+          nextRead = 0;
+        }
+        return current.get(nextRead++);
+      }
+
+      @Override
+      public void skipBytes(int count) {
+        setPosition(getPosition() + count);
+      }
+
+      @Override
+      public void readBytes(byte[] b, int offset, int len) {
+        while (len > 0) {
+          int chunkLeft = blockSize - nextRead;
+          if (len <= chunkLeft) {
+            arraycopy(current, nextRead, b, offset, len);
+            nextRead += len;
+            break;
+          } else {
+            if (chunkLeft > 0) {
+              arraycopy(current, nextRead, b, offset, chunkLeft);
+              offset += chunkLeft;
+              len -= chunkLeft;
+            }
+            current = blocks.get(nextBuffer++);
+            nextRead = 0;
+          }
+        }
+      }
+
+      @Override
+      public long getPosition() {
+        return ((long) nextBuffer - 1) * blockSize + nextRead;
+      }
+
+      @Override
+      public void setPosition(long pos) {
+        int bufferIndex = (int) (pos >> blockBits);
+        nextBuffer = bufferIndex + 1;
+        current = blocks.get(bufferIndex);
+        nextRead = (int) (pos & blockMask);
+        assert getPosition() == pos;
+      }
+
+      @Override
+      public boolean reversed() {
+        return false;
+      }
+    };
+  }
+
+  public FST.BytesReader getReverseReader() {
+    return getReverseReader(true);
+  }
+
+  FST.BytesReader getReverseReader(boolean allowSingle) {
+    if (allowSingle && blocks.size() == 1) {
+      return new ReverseBytesReaderByteArray(blocks.get(0));
+    }
+    return new FST.BytesReader() {
+      private ByteArray current = blocks.size() == 0 ? null : blocks.get(0);
+      private int nextBuffer = -1;
+      private int nextRead = 0;
+
+      @Override
+      public byte readByte() {
+        if (nextRead == -1) {
+          current = blocks.get(nextBuffer--);
+          nextRead = blockSize - 1;
+        }
+        return current.get(nextRead--);
+      }
+
+      @Override
+      public void skipBytes(int count) {
+        setPosition(getPosition() - count);
+      }
+
+      @Override
+      public void readBytes(byte[] b, int offset, int len) {
+        for (int i = 0; i < len; i++) {
+          b[offset + i] = readByte();
+        }
+      }
+
+      @Override
+      public long getPosition() {
+        return ((long) nextBuffer + 1) * blockSize + nextRead;
+      }
+
+      @Override
+      public void setPosition(long pos) {
+        // NOTE: a little weird because if you
+        // setPosition(0), the next byte you read is
+        // bytes[0] ... but I would expect bytes[-1] (ie,
+        // EOF)...?
+        int bufferIndex = (int) (pos >> blockBits);
+        nextBuffer = bufferIndex - 1;
+        current = blocks.get(bufferIndex);
+        nextRead = (int) (pos & blockMask);
+        assert getPosition() == pos : "pos=" + pos + " getPos()=" + getPosition();
+      }
+
+      @Override
+      public boolean reversed() {
+        return true;
+      }
+    };
+  }
+
+  static class ForwardBytesReaderByteArray extends FST.BytesReader {
+    private final ByteArray bytes;
+    private int pos;
+
+    public ForwardBytesReaderByteArray(ByteArray bytes) {
+      this.bytes = bytes;
+    }
+
+    @Override
+    public byte readByte() {
+      return bytes.get(pos++);
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) {
+      arraycopy(bytes, pos, b, offset, len);
+      pos += len;
+    }
+
+    @Override
+    public void skipBytes(int count) {
+      pos += count;
+    }
+
+    @Override
+    public long getPosition() {
+      return pos;
+    }
+
+    @Override
+    public void setPosition(long pos) {
+      this.pos = (int) pos;
+    }
+
+    @Override
+    public boolean reversed() {
+      return false;
+    }
+  }
+
+  static class ReverseBytesReaderByteArray extends FST.BytesReader {
+    private final ByteArray bytes;
+    private int pos;
+
+    public ReverseBytesReaderByteArray(ByteArray bytes) {
+      this.bytes = bytes;
+    }
+
+    @Override
+    public byte readByte() {
+      return bytes.get(pos--);
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) {
+      for (int i = 0; i < len; i++) {
+        b[offset + i] = bytes.get(pos--);
+      }
+    }
+
+    @Override
+    public void skipBytes(int count) {
+      pos -= count;
+    }
+
+    @Override
+    public long getPosition() {
+      return pos;
+    }
+
+    @Override
+    public void setPosition(long pos) {
+      this.pos = (int) pos;
+    }
+
+    @Override
+    public boolean reversed() {
+      return true;
+    }
+  }
+
+  public static void arraycopy(ByteArray src, int srcOffset, byte[] dest, int destOffset,
int len) {
+    src.get(srcOffset, dest, destOffset, len);
+  }
+
+  public static void arraycopy(ByteArray src, int srcOffset, ByteArray dest, int destOffset,
int len) {
+    src.copy(srcOffset, dest, destOffset, len);
+  }
+
+  public static void arraycopy(byte[] src, int srcOffset, ByteArray dest, int destOffset,
int len) {
+    dest.put(destOffset, src, srcOffset, len);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
----------------------------------------------------------------------
diff --git a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
index 7e20fe1..a6d7824 100644
--- a/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
+++ b/blur-util/src/main/java/org/apache/blur/utils/BlurConstants.java
@@ -129,6 +129,7 @@ public class BlurConstants {
   public static final String BLUR_CONTROLLER_THRIFT_ACCEPT_QUEUE_SIZE_PER_THREAD = "blur.controller.thrift.accept.queue.size.per.thread";
   public static final String BLUR_CLIENTPOOL_CLIENT_CLOSE_THRESHOLD = "blur.clientpool.client.close.threshold";
   public static final String BLUR_CLIENTPOOL_CLIENT_CLEAN_FREQUENCY = "blur.clientpool.client.clean.frequency";
+  public static final String BLUR_LUCENE_FST_BYTEARRAY_FACTORY = "blur.lucene.fst.bytearray.factory";
   
   public static final String BLUR_THRIFT_MAX_FRAME_SIZE = "blur.thrift.max.frame.size";
   

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/393c9f53/blur-util/src/main/resources/blur-default.properties
----------------------------------------------------------------------
diff --git a/blur-util/src/main/resources/blur-default.properties b/blur-util/src/main/resources/blur-default.properties
index 9a0e60c..10f8f77 100644
--- a/blur-util/src/main/resources/blur-default.properties
+++ b/blur-util/src/main/resources/blur-default.properties
@@ -184,6 +184,9 @@ blur.shard.blurindex.class=
 # Defines the blur read interceptor class that can mask data from query results as well as
data fetches.
 blur.shard.read.interceptor=
 
+# Defines the byte array factory class that blur will use to manage the FST trees in Lucene
(extends org.apache.blur.lucene.fst.ByteArrayFactory).
+blur.lucene.fst.bytearray.factory=
+
 
 ### Controller Server Configuration
 


Mime
View raw message