incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [25/51] [partial] Fixed BLUR-126.
Date Thu, 06 Jun 2013 18:57:57 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java
new file mode 100644
index 0000000..fba2a42
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/HdfsDirectoryTest.java
@@ -0,0 +1,206 @@
+package org.apache.blur.store;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.blur.store.buffer.BufferStore;
+import org.apache.blur.store.hdfs.HdfsDirectory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMDirectory;
+import org.junit.Before;
+import org.junit.Test;
+
+public class HdfsDirectoryTest {
+  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
+
+  private static final int MAX_NUMBER_OF_WRITES = 10000;
+  private static final int MIN_FILE_SIZE = 100;
+  private static final int MAX_FILE_SIZE = 100000;
+  private static final int MIN_BUFFER_SIZE = 1;
+  private static final int MAX_BUFFER_SIZE = 5000;
+  private static final int MAX_NUMBER_OF_READS = 10000;
+  private HdfsDirectory directory;
+  private File file;
+  private long seed;
+  private Random random;
+
+  @Before
+  public void setUp() throws IOException {
+    BufferStore.init(128, 128);
+    file = new File(TMPDIR, "hdfsdirectorytest");
+    rm(file);
+    URI uri = new File(file, "hdfs").toURI();
+    Path hdfsDirPath = new Path(uri.toString());
+    Configuration conf = new Configuration();
+    directory = new HdfsDirectory(conf, hdfsDirPath);
+    seed = new Random().nextLong();
+    random = new Random(seed);
+  }
+
+  @Test
+  public void testWritingAndReadingAFile() throws IOException {
+
+    IndexOutput output = directory.createOutput("testing.test", IOContext.DEFAULT);
+    output.writeInt(12345);
+    output.flush();
+    output.close();
+
+    IndexInput input = directory.openInput("testing.test", IOContext.DEFAULT);
+    assertEquals(12345, input.readInt());
+    input.close();
+
+    String[] listAll = directory.listAll();
+    assertEquals(1, listAll.length);
+    assertEquals("testing.test", listAll[0]);
+
+    assertEquals(4, directory.fileLength("testing.test"));
+
+    IndexInput input1 = directory.openInput("testing.test", IOContext.DEFAULT);
+
+    IndexInput input2 = (IndexInput) input1.clone();
+    assertEquals(12345, input2.readInt());
+    input2.close();
+
+    assertEquals(12345, input1.readInt());
+    input1.close();
+
+    assertFalse(directory.fileExists("testing.test.other"));
+    assertTrue(directory.fileExists("testing.test"));
+    directory.deleteFile("testing.test");
+    assertFalse(directory.fileExists("testing.test"));
+  }
+
+  @Test
+  public void testEOF() throws IOException {
+    Directory fsDir = new RAMDirectory();
+    String name = "test.eof";
+    createFile(name, fsDir, directory);
+    long fsLength = fsDir.fileLength(name);
+    long hdfsLength = directory.fileLength(name);
+    assertEquals(fsLength, hdfsLength);
+    testEof(name, fsDir, fsLength);
+    testEof(name, directory, hdfsLength);
+  }
+
+  private void testEof(String name, Directory directory, long length) throws IOException {
+    IndexInput input = directory.openInput(name, IOContext.DEFAULT);
+    input.seek(length);
+    try {
+      input.readByte();
+      fail("should throw eof");
+    } catch (IOException e) {
+    }
+  }
+
+  @Test
+  public void testWrites() throws IOException {
+    int i = 0;
+    try {
+      Set<String> names = new HashSet<String>();
+      for (; i < 10; i++) {
+        Directory fsDir = new RAMDirectory();
+        String name = getName();
+        System.out.println("Working on pass [" + i + "] seed [" + seed + "] contains [" + names.contains(name) + "]");
+        names.add(name);
+        createFile(name, fsDir, directory);
+        assertInputsEquals(name, fsDir, directory);
+        fsDir.close();
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail("Test failed with seed [" + seed + "] on pass [" + i + "]");
+    }
+  }
+
+  private void assertInputsEquals(String name, Directory fsDir, HdfsDirectory hdfs) throws IOException {
+    int reads = random.nextInt(MAX_NUMBER_OF_READS);
+    IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT);
+    IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT);
+    assertEquals(fsInput.length(), hdfsInput.length());
+    int fileLength = (int) fsInput.length();
+    for (int i = 0; i < reads; i++) {
+      byte[] fsBuf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE];
+      byte[] hdfsBuf = new byte[fsBuf.length];
+      int offset = random.nextInt(fsBuf.length);
+      int length = random.nextInt(fsBuf.length - offset);
+      int pos = random.nextInt(fileLength - length);
+      fsInput.seek(pos);
+      fsInput.readBytes(fsBuf, offset, length);
+      hdfsInput.seek(pos);
+      hdfsInput.readBytes(hdfsBuf, offset, length);
+      for (int f = offset; f < length; f++) {
+        if (fsBuf[f] != hdfsBuf[f]) {
+          fail();
+        }
+      }
+    }
+    fsInput.close();
+    hdfsInput.close();
+  }
+
+  private void createFile(String name, Directory fsDir, HdfsDirectory hdfs) throws IOException {
+    int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
+    int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
+    IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT);
+    fsOutput.setLength(fileLength);
+    IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT);
+    hdfsOutput.setLength(fileLength);
+    for (int i = 0; i < writes; i++) {
+      byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE];
+      random.nextBytes(buf);
+      int offset = random.nextInt(buf.length);
+      int length = random.nextInt(buf.length - offset);
+      fsOutput.writeBytes(buf, offset, length);
+      hdfsOutput.writeBytes(buf, offset, length);
+    }
+    fsOutput.close();
+    hdfsOutput.close();
+  }
+
+  private String getName() {
+    return Long.toString(Math.abs(random.nextLong()));
+  }
+
+  public static void rm(File file) {
+    if (!file.exists()) {
+      return;
+    }
+    if (file.isDirectory()) {
+      for (File f : file.listFiles()) {
+        rm(f);
+      }
+    }
+    file.delete();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
new file mode 100644
index 0000000..b1d9d12
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockCacheTest.java
@@ -0,0 +1,147 @@
+package org.apache.blur.store.blockcache;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Test;
+
+public class BlockCacheTest {
+  @Test
+  public void testBlockCache() throws IOException, InterruptedException, ExecutionException {
+    int blocksPerSlab = 1024;
+    int slabs = 4;
+    // Test block are larger than cache size.
+    final int blocksInTest = (blocksPerSlab * slabs) * 2;
+    final int blockSize = BlockCache._8K;
+    int slabSize = blockSize * blocksPerSlab;
+    long totalMemory = slabs * (long) slabSize;
+
+    final BlockCache blockCache = new BlockCache(true, totalMemory, slabSize);
+    final int threads = 4;
+    ExecutorService pool = Executors.newFixedThreadPool(threads);
+    List<Future<Boolean>> futures = new ArrayList<Future<Boolean>>();
+    final int testSpace = blocksInTest / threads;
+    for (int g = 0; g < threads; g++) {
+      final int file = g;
+      futures.add(pool.submit(new Callable<Boolean>() {
+        @Override
+        public Boolean call() throws Exception {
+          return runTest(blockSize, file, testSpace, blockCache);
+        }
+      }));
+    }
+
+    for (Future<Boolean> future : futures) {
+      if (!future.get()) {
+        fail();
+      }
+    }
+    blockCache.close();
+  }
+
+  private boolean runTest(int blockSize, int file, int blocksInTest, BlockCache blockCache) {
+    byte[] buffer = new byte[blockSize];
+    Random random = new Random();
+
+    byte[] newData = new byte[blockSize];
+    AtomicLong hitsInCache = new AtomicLong();
+    AtomicLong missesInCache = new AtomicLong();
+    long storeTime = 0;
+    long fetchTime = 0;
+    int passes = 10000;
+
+    BlockCacheKey blockCacheKey = new BlockCacheKey();
+
+    for (int j = 0; j < passes; j++) {
+      long block = random.nextInt(blocksInTest);
+      blockCacheKey.setBlock(block);
+      blockCacheKey.setFile(file);
+
+      if (blockCache.fetch(blockCacheKey, buffer)) {
+        hitsInCache.incrementAndGet();
+      } else {
+        missesInCache.incrementAndGet();
+      }
+
+      byte[] testData = testData(random, blockSize, newData);
+      long t1 = System.nanoTime();
+      boolean store = blockCache.store(blockCacheKey, 0, testData, 0, blockSize);
+      storeTime += (System.nanoTime() - t1);
+
+      if (store) {
+        long t3 = System.nanoTime();
+        if (blockCache.fetch(blockCacheKey, buffer)) {
+          fetchTime += (System.nanoTime() - t3);
+          if (!Arrays.equals(testData, buffer)) {
+            return false;
+          }
+        }
+      }
+    }
+    System.out.println("Cache Hits    = " + hitsInCache.get());
+    System.out.println("Cache Misses  = " + missesInCache.get());
+    System.out.println("Store         = avg " + (storeTime / (double) passes) / 1000000.0 + " ms");
+    System.out.println("Fetch         = avg " + (fetchTime / (double) passes) / 1000000.0 + " ms");
+    System.out.println("# of Elements = " + blockCache.getSize());
+    return true;
+  }
+
+  /**
+   * Verify checking of buffer size limits against the cached block size.
+   */
+  @Test
+  public void testLongBuffer() {
+    Random random = new Random();
+    int blockSize = BlockCache._8K;
+    int slabSize = blockSize * 1024;
+    long totalMemory = 2 * slabSize;
+
+    BlockCache blockCache = new BlockCache(true, totalMemory, slabSize);
+    BlockCacheKey blockCacheKey = new BlockCacheKey();
+    blockCacheKey.setBlock(0);
+    blockCacheKey.setFile(0);
+    byte[] newData = new byte[blockSize * 3];
+    byte[] testData = testData(random, blockSize, newData);
+
+    assertTrue(blockCache.store(blockCacheKey, 0, testData, 0, blockSize));
+    assertTrue(blockCache.store(blockCacheKey, 0, testData, blockSize, blockSize));
+    assertTrue(blockCache.store(blockCacheKey, 0, testData, blockSize * 2, blockSize));
+
+    assertTrue(blockCache.store(blockCacheKey, 1, testData, 0, blockSize - 1));
+    assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize, blockSize - 1));
+    assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize * 2, blockSize - 1));
+  }
+
+  private static byte[] testData(Random random, int size, byte[] buf) {
+    random.nextBytes(buf);
+    return buf;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryCacheTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryCacheTest.java b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryCacheTest.java
new file mode 100644
index 0000000..6a30d10
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryCacheTest.java
@@ -0,0 +1,38 @@
+package org.apache.blur.store.blockcache;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.junit.Before;
+import org.junit.Test;
+
+public class BlockDirectoryCacheTest {
+  private BlockCache blockCache;
+  private BlockDirectoryCache blockDirectoryCache;
+
+  @Before
+  public void setup() {
+    int slabSize = BlockCache._8K * 1024;
+    long totalMemory = 2 * slabSize;
+    blockCache = new BlockCache(true, totalMemory, slabSize);
+    blockDirectoryCache = new BlockDirectoryCache(blockCache);
+  }
+
+  @Test
+  public void validateEmptyOutputFile() {
+    blockDirectoryCache.renameCacheFile("foo", "bar");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java
----------------------------------------------------------------------
diff --git a/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java
new file mode 100644
index 0000000..f2ef236
--- /dev/null
+++ b/blur-store/src/test/java/org/apache/blur/store/blockcache/BlockDirectoryTest.java
@@ -0,0 +1,228 @@
+package org.apache.blur.store.blockcache;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Random;
+
+import javax.jws.Oneway;
+
+import org.apache.blur.store.blockcache.BlockDirectory;
+import org.apache.blur.store.blockcache.Cache;
+import org.apache.blur.store.buffer.BufferStore;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+
+public class BlockDirectoryTest {
+  private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "/tmp"));
+
+  private class MapperCache implements Cache {
+    public Map<String, byte[]> map = new ConcurrentLinkedHashMap.Builder<String, byte[]>().maximumWeightedCapacity(8).build();
+
+    @Override
+    public void update(String name, long blockId, int blockOffset, byte[] buffer, int offset, int length) {
+      byte[] cached = map.get(name + blockId);
+      if (cached != null) {
+        int newlen = Math.max(cached.length, blockOffset + length);
+        byte[] b = new byte[newlen];
+        System.arraycopy(cached, 0, b, 0, cached.length);
+        System.arraycopy(buffer, offset, b, blockOffset, length);
+        cached = b;
+      } else {
+        cached = copy(blockOffset, buffer, offset, length);
+      }
+      map.put(name + blockId, cached);
+    }
+
+    private byte[] copy(int blockOffset, byte[] buffer, int offset, int length) {
+      byte[] b = new byte[length + blockOffset];
+      System.arraycopy(buffer, offset, b, blockOffset, length);
+      return b;
+    }
+
+    @Override
+    public boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) {
+      // return false;
+      byte[] data = map.get(name + blockId);
+      if (data == null) {
+        return false;
+      }
+      System.arraycopy(data, blockOffset, b, off, lengthToReadInBlock);
+      return true;
+    }
+
+    @Override
+    public void delete(String name) {
+
+    }
+
+    @Override
+    public long size() {
+      return map.size();
+    }
+
+    @Override
+    public void renameCacheFile(String source, String dest) {
+    }
+  }
+
+  private static final int MAX_NUMBER_OF_WRITES = 10000;
+  private static final int MIN_FILE_SIZE = 100;
+  private static final int MAX_FILE_SIZE = 100000;
+  private static final int MIN_BUFFER_SIZE = 1;
+  private static final int MAX_BUFFER_SIZE = 12000;
+  private static final int MAX_NUMBER_OF_READS = 20000;
+  private Directory directory;
+  private File file;
+  private long seed;
+  private Random random;
+  private MapperCache mapperCache;
+  
+  @Before
+  public void setUp() throws IOException {
+    BufferStore.init(128, 128);
+    file = new File(TMPDIR, "blockdirectorytest");
+    rm(file);
+    file.mkdirs();
+    FSDirectory dir = FSDirectory.open(new File(file, "base"));
+    mapperCache = new MapperCache();
+    directory = new BlockDirectory("test", dir, mapperCache);
+    seed = new Random().nextLong();
+    System.out.println("Seed is " + seed);
+    random = new Random(seed);
+  }
+
+  @Test
+  public void testEOF() throws IOException {
+    Directory fsDir = FSDirectory.open(new File(file, "normal"));
+    String name = "test.eof";
+    createFile(name, fsDir, directory);
+    long fsLength = fsDir.fileLength(name);
+    long hdfsLength = directory.fileLength(name);
+    assertEquals(fsLength, hdfsLength);
+    testEof(name, fsDir, fsLength);
+    testEof(name, directory, hdfsLength);
+  }
+
+  private void testEof(String name, Directory directory, long length) throws IOException {
+    IndexInput input = directory.openInput(name, IOContext.DEFAULT);
+    input.seek(length);
+    try {
+      input.readByte();
+      fail("should throw eof");
+    } catch (IOException e) {
+    }
+  }
+
+  @Test
+  public void testRandomAccessWrites() throws IOException {
+    long t1 = System.nanoTime();
+
+    int i = 0;
+    try {
+      for (; i < 10; i++) {
+        Directory fsDir = FSDirectory.open(new File(file, "normal"));
+        String name = getName();
+        createFile(name, fsDir, directory);
+        assertInputsEquals(name, fsDir, directory);
+      }
+    } catch (Exception e) {
+      e.printStackTrace();
+      fail("Test failed with seed [" + seed + "] on pass [" + i + "]");
+    }
+    long t2 = System.nanoTime();
+    System.out.println("Total time is " + ((t2 - t1)/1000000) + "ms");
+  }
+
+  @Test
+  public void testRandomAccessWritesLargeCache() throws IOException {
+    mapperCache.map = new ConcurrentLinkedHashMap.Builder<String, byte[]>().maximumWeightedCapacity(10000).build();
+    testRandomAccessWrites();
+  }
+
+  private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException {
+    int reads = random.nextInt(MAX_NUMBER_OF_READS);
+    IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT);
+    IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT);
+    assertEquals(fsInput.length(), hdfsInput.length());
+    int fileLength = (int) fsInput.length();
+    for (int i = 0; i < reads; i++) {
+      byte[] fsBuf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE];
+      byte[] hdfsBuf = new byte[fsBuf.length];
+      int offset = random.nextInt(fsBuf.length);
+      int length = random.nextInt(fsBuf.length - offset);
+      int pos = random.nextInt(fileLength - length);
+      fsInput.seek(pos);
+      fsInput.readBytes(fsBuf, offset, length);
+      hdfsInput.seek(pos);
+      hdfsInput.readBytes(hdfsBuf, offset, length);
+      for (int f = offset; f < length; f++) {
+        if (fsBuf[f] != hdfsBuf[f]) {
+          fail(Long.toString(seed) + " read [" + i + "]");
+        }
+      }
+    }
+    fsInput.close();
+    hdfsInput.close();
+  }
+
+  private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException {
+    int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
+    int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
+    IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT);
+    IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT);
+    for (int i = 0; i < writes; i++) {
+      byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE];
+      random.nextBytes(buf);
+      int offset = random.nextInt(buf.length);
+      int length = random.nextInt(buf.length - offset);
+      fsOutput.writeBytes(buf, offset, length);
+      hdfsOutput.writeBytes(buf, offset, length);
+    }
+    fsOutput.close();
+    hdfsOutput.close();
+  }
+
+  private String getName() {
+    return Long.toString(Math.abs(random.nextLong()));
+  }
+
+  public static void rm(File file) {
+    if (!file.exists()) {
+      return;
+    }
+    if (file.isDirectory()) {
+      for (File f : file.listFiles()) {
+        rm(f);
+      }
+    }
+    file.delete();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/pom.xml
----------------------------------------------------------------------
diff --git a/blur-thrift/pom.xml b/blur-thrift/pom.xml
new file mode 100644
index 0000000..58c7a96
--- /dev/null
+++ b/blur-thrift/pom.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor 
+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	you under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.apache.blur</groupId>
+		<artifactId>blur</artifactId>
+		<version>0.1.5</version>
+		<relativePath>../pom.xml</relativePath>
+	</parent>
+	<groupId>org.apache.blur</groupId>
+	<artifactId>blur-thrift</artifactId>
+	<packaging>jar</packaging>
+	<name>Blur Thrift</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.blur</groupId>
+			<artifactId>blur-util</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>${log4j.version}</version>
+			<scope>provided</scope>
+			<exclusions>
+				<exclusion>
+					<groupId>javax.mail</groupId>
+					<artifactId>mail</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>javax.jms</groupId>
+					<artifactId>jms</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>com.sun.jdmk</groupId>
+					<artifactId>jmxtools</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>com.sun.jmx</groupId>
+					<artifactId>jmxri</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.httpcomponents</groupId>
+			<artifactId>httpclient</artifactId>
+			<version>${httpclient.version}</version>
+		</dependency>
+	</dependencies>
+
+	<repositories>
+		<repository>
+			<id>libdir</id>
+			<url>file://${basedir}/../lib</url>
+		</repository>
+	</repositories>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<configuration>
+					<source>1.6</source>
+					<target>1.6</target>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>test-jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/EncodingUtils.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/EncodingUtils.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/EncodingUtils.java
new file mode 100644
index 0000000..9f17528
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/EncodingUtils.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+/**
+ * Utility methods for use when encoding/decoding raw data as byte arrays.
+ */
+public class EncodingUtils {
+
+  /**
+   * Encode <code>integer</code> as a series of 4 bytes into <code>buf</code>
+   * starting at position 0 within that buffer.
+   * 
+   * @param integer
+   *          The integer to encode.
+   * @param buf
+   *          The buffer to write to.
+   */
+  public static final void encodeBigEndian(final int integer, final byte[] buf) {
+    encodeBigEndian(integer, buf, 0);
+  }
+
+  /**
+   * Encode <code>integer</code> as a series of 4 bytes into <code>buf</code>
+   * starting at position <code>offset</code>.
+   * 
+   * @param integer
+   *          The integer to encode.
+   * @param buf
+   *          The buffer to write to.
+   * @param offset
+   *          The offset within <code>buf</code> to start the encoding.
+   */
+  public static final void encodeBigEndian(final int integer, final byte[] buf, int offset) {
+    buf[offset] = (byte) (0xff & (integer >> 24));
+    buf[offset + 1] = (byte) (0xff & (integer >> 16));
+    buf[offset + 2] = (byte) (0xff & (integer >> 8));
+    buf[offset + 3] = (byte) (0xff & (integer));
+  }
+
+  /**
+   * Decode a series of 4 bytes from <code>buf</code>, starting at position 0,
+   * and interpret them as an integer.
+   * 
+   * @param buf
+   *          The buffer to read from.
+   * @return An integer, as read from the buffer.
+   */
+  public static final int decodeBigEndian(final byte[] buf) {
+    return decodeBigEndian(buf, 0);
+  }
+
+  /**
+   * Decode a series of 4 bytes from <code>buf</code>, start at
+   * <code>offset</code>, and interpret them as an integer.
+   * 
+   * @param buf
+   *          The buffer to read from.
+   * @param offset
+   *          The offset with <code>buf</code> to start the decoding.
+   * @return An integer, as read from the buffer.
+   */
+  public static final int decodeBigEndian(final byte[] buf, int offset) {
+    return ((buf[offset] & 0xff) << 24) | ((buf[offset + 1] & 0xff) << 16)
+        | ((buf[offset + 2] & 0xff) << 8) | ((buf[offset + 3] & 0xff));
+  }
+
+  /**
+   * Bitfield utilities.
+   * Returns true if the bit at position is set in v.
+   */
+  public static final boolean testBit(byte v, int position) {
+    return testBit((int)v, position);
+  }
+
+  public static final boolean testBit(short v, int position) {
+    return testBit((int)v, position);
+  }
+
+  public static final boolean testBit(int v, int position) {
+    return (v & (1 << position)) != 0;
+  }
+
+  public static final boolean testBit(long v, int position) {
+    return (v & (1L << position)) != 0L;
+  }
+
+  /**
+   * Returns v, with the bit at position set to zero.
+   */
+  public static final byte clearBit(byte v, int position) {
+    return (byte)clearBit((int)v, position);
+  }
+
+  public static final short clearBit(short v, int position) {
+    return (short)clearBit((int)v, position);
+  }
+
+  public static final int clearBit(int v, int position) {
+    return v & ~(1 << position);
+  }
+
+  public static final long clearBit(long v, int position) {
+    return v & ~(1L << position);
+  }
+
+  /**
+   * Returns v, with the bit at position set to 1 or 0 depending on value.
+   */
+  public static final byte setBit(byte v, int position, boolean value) {
+    return (byte)setBit((int)v, position, value);
+  }
+
+  public static final short setBit(short v, int position, boolean value) {
+    return (short)setBit((int)v, position, value);
+  }
+
+  public static final int setBit(int v, int position, boolean value) {
+    if(value)
+      return v | (1 << position);
+    else
+      return clearBit(v, position);
+  }
+
+  public static final long setBit(long v, int position, boolean value) {
+    if(value)
+      return v | (1L << position);
+    else
+      return clearBit(v, position);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ProcessFunction.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ProcessFunction.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ProcessFunction.java
new file mode 100644
index 0000000..fe0670b
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ProcessFunction.java
@@ -0,0 +1,68 @@
+/**
+ * 
+ */
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessage;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessageType;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ProcessFunction<I, T extends TBase> {
+  private final String methodName;
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ProcessFunction.class.getName());
+
+  public ProcessFunction(String methodName) {
+    this.methodName = methodName;
+  }
+
+  public final void process(int seqid, TProtocol iprot, TProtocol oprot, I iface) throws TException {
+    T args = getEmptyArgsInstance();
+    try {
+      args.read(iprot);
+    } catch (TProtocolException e) {
+      iprot.readMessageEnd();
+      TApplicationException x = new TApplicationException(TApplicationException.PROTOCOL_ERROR, e.getMessage());
+      oprot.writeMessageBegin(new TMessage(getMethodName(), TMessageType.EXCEPTION, seqid));
+      x.write(oprot);
+      oprot.writeMessageEnd();
+      oprot.getTransport().flush();
+      return;
+    }
+    iprot.readMessageEnd();
+    TBase result = null;
+
+    try {
+      result = getResult(iface, args);
+    } catch(Throwable th) {
+      LOGGER.error("Internal error processing " + getMethodName(), th);
+      TApplicationException x = new TApplicationException(TApplicationException.INTERNAL_ERROR, 
+        "Internal error processing " + getMethodName());
+      oprot.writeMessageBegin(new TMessage(getMethodName(), TMessageType.EXCEPTION, seqid));
+      x.write(oprot);
+      oprot.writeMessageEnd();
+      oprot.getTransport().flush();
+      return;
+    }
+
+    if(!isOneway()) {
+      oprot.writeMessageBegin(new TMessage(getMethodName(), TMessageType.REPLY, seqid));
+      result.write(oprot);
+      oprot.writeMessageEnd();
+      oprot.getTransport().flush();
+    }
+  }
+
+  protected abstract boolean isOneway();
+
+  public abstract TBase getResult(I iface, T args) throws TException;
+
+  public abstract T getEmptyArgsInstance();
+
+  public String getMethodName() {
+    return methodName;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ShortStack.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ShortStack.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ShortStack.java
new file mode 100644
index 0000000..d7a01f5
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/ShortStack.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+/**
+ * ShortStack is a short-specific Stack implementation written for the express
+ * purpose of very fast operations on TCompactProtocol's field id stack. This
+ * implementation performs at least 10x faster than java.util.Stack.
+ */
+public class ShortStack {
+
+  private short[] vector;
+  private int top = -1;
+
+  public ShortStack(int initialCapacity) {
+    vector = new short[initialCapacity];
+  }
+
+  public short pop() {
+    return vector[top--];
+  }
+
+  public void push(short pushed) {
+    if (vector.length == top + 1) {
+      grow();
+    }
+    vector[++top] = pushed;
+  }
+
+  private void grow() {
+    short[] newVector = new short[vector.length * 2];
+    System.arraycopy(vector, 0, newVector, 0, vector.length);
+    vector = newVector;
+  }
+
+  public short peek() {
+    return vector[top];
+  }
+
+  public void clear() {
+    top = -1;
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("<ShortStack vector:[");
+    for (int i = 0; i < vector.length; i++) {
+      if (i != 0) {
+        sb.append(" ");
+      }
+
+      if (i == top) {
+        sb.append(">>");
+      }
+
+      sb.append(vector[i]);
+
+      if (i == top) {
+        sb.append("<<");
+      }
+    }
+    sb.append("]>");
+    return sb.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TApplicationException.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TApplicationException.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TApplicationException.java
new file mode 100644
index 0000000..51d24bd
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TApplicationException.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TStruct;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType;
+
+/**
+ * Application level exception
+ *
+ */
+public class TApplicationException extends TException {
+
+  private static final TStruct TAPPLICATION_EXCEPTION_STRUCT = new TStruct("TApplicationException");
+  private static final TField MESSAGE_FIELD = new TField("message", TType.STRING, (short)1);
+  private static final TField TYPE_FIELD = new TField("type", TType.I32, (short)2);
+
+  private static final long serialVersionUID = 1L;
+
+  public static final int UNKNOWN = 0;
+  public static final int UNKNOWN_METHOD = 1;
+  public static final int INVALID_MESSAGE_TYPE = 2;
+  public static final int WRONG_METHOD_NAME = 3;
+  public static final int BAD_SEQUENCE_ID = 4;
+  public static final int MISSING_RESULT = 5;
+  public static final int INTERNAL_ERROR = 6;
+  public static final int PROTOCOL_ERROR = 7;
+
+  protected int type_ = UNKNOWN;
+
+  public TApplicationException() {
+    super();
+  }
+
+  public TApplicationException(int type) {
+    super();
+    type_ = type;
+  }
+
+  public TApplicationException(int type, String message) {
+    super(message);
+    type_ = type;
+  }
+
+  public TApplicationException(String message) {
+    super(message);
+  }
+
+  public int getType() {
+    return type_;
+  }
+
+  public static TApplicationException read(TProtocol iprot) throws TException {
+    TField field;
+    iprot.readStructBegin();
+
+    String message = null;
+    int type = UNKNOWN;
+
+    while (true) {
+      field = iprot.readFieldBegin();
+      if (field.type == TType.STOP) {
+        break;
+      }
+      switch (field.id) {
+      case 1:
+        if (field.type == TType.STRING) {
+          message = iprot.readString();
+        } else {
+          TProtocolUtil.skip(iprot, field.type);
+        }
+        break;
+      case 2:
+        if (field.type == TType.I32) {
+          type = iprot.readI32();
+        } else {
+          TProtocolUtil.skip(iprot, field.type);
+        }
+        break;
+      default:
+        TProtocolUtil.skip(iprot, field.type);
+        break;
+      }
+      iprot.readFieldEnd();
+    }
+    iprot.readStructEnd();
+
+    return new TApplicationException(type, message);
+  }
+
+  public void write(TProtocol oprot) throws TException {
+    oprot.writeStructBegin(TAPPLICATION_EXCEPTION_STRUCT);
+    if (getMessage() != null) {
+      oprot.writeFieldBegin(MESSAGE_FIELD);
+      oprot.writeString(getMessage());
+      oprot.writeFieldEnd();
+    }
+    oprot.writeFieldBegin(TYPE_FIELD);
+    oprot.writeI32(type_);
+    oprot.writeFieldEnd();
+    oprot.writeFieldStop();
+    oprot.writeStructEnd();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBase.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBase.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBase.java
new file mode 100644
index 0000000..0676029
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBase.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import java.io.Serializable;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
+
+/**
+ * Generic base interface for generated Thrift objects.
+ *
+ */
+public interface TBase<T extends TBase<?,?>, F extends TFieldIdEnum> extends Comparable<T>,  Serializable {
+
+  /**
+   * Reads the TObject from the given input protocol.
+   *
+   * @param iprot Input protocol
+   */
+  public void read(TProtocol iprot) throws TException;
+
+  /**
+   * Writes the objects out to the protocol
+   *
+   * @param oprot Output protocol
+   */
+  public void write(TProtocol oprot) throws TException;
+
+  /**
+   * Get the F instance that corresponds to fieldId.
+   */
+  public F fieldForId(int fieldId);
+
+  /**
+   * Check if a field is currently set or unset.
+   *
+   * @param field
+   */
+  public boolean isSet(F field);
+
+  /**
+   * Get a field's value by field variable. Primitive types will be wrapped in 
+   * the appropriate "boxed" types.
+   *
+   * @param field
+   */
+  public Object getFieldValue(F field);
+
+  /**
+   * Set a field's value by field variable. Primitive types must be "boxed" in
+   * the appropriate object wrapper type.
+   *
+   * @param field
+   */
+  public void setFieldValue(F field, Object value);
+
+  public TBase<T, F> deepCopy();
+
+  /**
+   * Return to the state of having just been initialized, as though you had just
+   * called the default constructor.
+   */
+  public void clear();
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseHelper.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseHelper.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseHelper.java
new file mode 100644
index 0000000..ff52bfe
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseHelper.java
@@ -0,0 +1,306 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+public final class TBaseHelper {
+
+  private TBaseHelper(){}
+
+  private static final Comparator comparator = new NestedStructureComparator();
+
+  public static int compareTo(Object o1, Object o2) {
+    if (o1 instanceof Comparable) {
+      return compareTo((Comparable)o1, (Comparable)o2);
+    } else if (o1 instanceof List) {
+      return compareTo((List)o1, (List)o2);
+    } else if (o1 instanceof Set) {
+      return compareTo((Set)o1, (Set)o2);
+    } else if (o1 instanceof Map) {
+      return compareTo((Map)o1, (Map)o2);
+    } else if (o1 instanceof byte[]) {
+      return compareTo((byte[])o1, (byte[])o2);
+    } else {
+      throw new IllegalArgumentException("Cannot compare objects of type " + o1.getClass());
+    }
+  }
+
+  public static int compareTo(boolean a, boolean b) {
+    return Boolean.valueOf(a).compareTo(b);
+  }
+
+  public static int compareTo(byte a, byte b) {
+    if (a < b) {
+      return -1;
+    } else if (b < a) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  public static int compareTo(short a, short b) {
+    if (a < b) {
+      return -1;
+    } else if (b < a) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  public static int compareTo(int a, int b) {
+    if (a < b) {
+      return -1;
+    } else if (b < a) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  public static int compareTo(long a, long b) {
+    if (a < b) {
+      return -1;
+    } else if (b < a) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  public static int compareTo(double a, double b) {
+    if (a < b) {
+      return -1;
+    } else if (b < a) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+
+  public static int compareTo(String a, String b) {
+    return a.compareTo(b);
+  }
+
+  public static int compareTo(byte[] a, byte[] b) {
+    int sizeCompare = compareTo(a.length, b.length);
+    if (sizeCompare != 0) {
+      return sizeCompare;
+    }
+    for (int i = 0; i < a.length; i++) {
+      int byteCompare = compareTo(a[i], b[i]);
+      if (byteCompare != 0) {
+        return byteCompare;
+      }
+    }
+    return 0;
+  }
+
+  public static int compareTo(Comparable a, Comparable b) {
+    return a.compareTo(b);
+  }
+
+  public static int compareTo(List a, List b) {
+    int lastComparison = compareTo(a.size(), b.size());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    for (int i = 0; i < a.size(); i++) {
+      lastComparison = comparator.compare(a.get(i), b.get(i));
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+    return 0;
+  }
+
+  public static int compareTo(Set a, Set b) {
+    int lastComparison = compareTo(a.size(), b.size());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+    SortedSet sortedA = new TreeSet(comparator);
+    sortedA.addAll(a);
+    SortedSet sortedB = new TreeSet(comparator);
+    sortedB.addAll(b);
+
+    Iterator iterA = sortedA.iterator();
+    Iterator iterB = sortedB.iterator();
+
+    // Compare each item.
+    while (iterA.hasNext() && iterB.hasNext()) {
+      lastComparison = comparator.compare(iterA.next(), iterB.next());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+
+    return 0;
+  }
+
+  public static int compareTo(Map a, Map b) {
+    int lastComparison = compareTo(a.size(), b.size());
+    if (lastComparison != 0) {
+      return lastComparison;
+    }
+
+    // Sort a and b so we can compare them.
+    SortedMap sortedA = new TreeMap(comparator);
+    sortedA.putAll(a);
+    Iterator<Map.Entry> iterA = sortedA.entrySet().iterator();
+    SortedMap sortedB = new TreeMap(comparator);
+    sortedB.putAll(b);
+    Iterator<Map.Entry> iterB = sortedB.entrySet().iterator();
+
+    // Compare each item.
+    while (iterA.hasNext() && iterB.hasNext()) {
+      Map.Entry entryA = iterA.next();
+      Map.Entry entryB = iterB.next();
+      lastComparison = comparator.compare(entryA.getKey(), entryB.getKey());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+      lastComparison = comparator.compare(entryA.getValue(), entryB.getValue());
+      if (lastComparison != 0) {
+        return lastComparison;
+      }
+    }
+
+    return 0;
+  }
+
+  /**
+   * Comparator to compare items inside a structure (e.g. a list, set, or map).
+   */
+  private static class NestedStructureComparator implements Comparator {
+    public int compare(Object oA, Object oB) {
+      if (oA == null && oB == null) {
+        return 0;
+      } else if (oA == null) {
+        return -1;
+      } else if (oB == null) {
+        return 1;
+      } else if (oA instanceof List) {
+        return compareTo((List)oA, (List)oB);
+      } else if (oA instanceof Set) {
+        return compareTo((Set)oA, (Set)oB);
+      } else if (oA instanceof Map) {
+        return compareTo((Map)oA, (Map)oB);
+      } else if (oA instanceof byte[]) {
+        return compareTo((byte[])oA, (byte[])oB);
+      } else {
+        return compareTo((Comparable)oA, (Comparable)oB);
+      }
+    }
+  }
+
+  public static void toString(ByteBuffer bb, StringBuilder sb) {
+    byte[] buf = bb.array();
+
+    int arrayOffset = bb.arrayOffset();
+    int offset = arrayOffset + bb.position();
+    int origLimit = arrayOffset + bb.limit();
+    int limit = (origLimit - offset > 128) ? offset + 128 : origLimit;
+
+    for (int i = offset; i < limit; i++) {
+      if (i > offset) {
+        sb.append(" ");
+      }
+      sb.append(paddedByteString(buf[i]));
+    }
+    if (origLimit != limit) {
+      sb.append("...");
+    }
+  }
+
+  public static String paddedByteString(byte b) {
+    int extended = (b | 0x100) & 0x1ff;
+    return Integer.toHexString(extended).toUpperCase().substring(1);
+  }
+
+  public static byte[] byteBufferToByteArray(ByteBuffer byteBuffer) {
+    if (wrapsFullArray(byteBuffer)) {
+      return byteBuffer.array();
+    }
+    byte[] target = new byte[byteBuffer.remaining()];
+    byteBufferToByteArray(byteBuffer, target, 0);
+    return target;
+  }
+
+  public static boolean wrapsFullArray(ByteBuffer byteBuffer) {
+    return byteBuffer.hasArray()
+      && byteBuffer.position() == 0
+      && byteBuffer.arrayOffset() == 0
+      && byteBuffer.remaining() == byteBuffer.capacity();
+  }
+
+  public static int byteBufferToByteArray(ByteBuffer byteBuffer, byte[] target, int offset) {
+    int remaining = byteBuffer.remaining();
+    System.arraycopy(byteBuffer.array(),
+        byteBuffer.arrayOffset() + byteBuffer.position(),
+        target,
+        offset,
+        remaining);
+    return remaining;
+  }
+
+  public static ByteBuffer rightSize(ByteBuffer in) {
+    if (in == null) {
+      return null;
+    }
+    if (wrapsFullArray(in)) {
+      return in;
+    }
+    return ByteBuffer.wrap(byteBufferToByteArray(in));
+  }
+
+  public static ByteBuffer copyBinary(final ByteBuffer orig) {
+    if (orig == null) {
+      return null;
+    }
+    ByteBuffer copy = ByteBuffer.wrap(new byte[orig.remaining()]);
+    if (orig.hasArray()) {
+      System.arraycopy(orig.array(), orig.arrayOffset() + orig.position(), copy.array(), 0, orig.remaining());
+    } else {
+      orig.slice().get(copy.array());
+    }
+
+    return copy;
+  }
+
+  public static byte[] copyBinary(final byte[] orig) {
+    if (orig == null) {
+      return null;
+    }
+
+    byte[] copy = new byte[orig.length];
+    System.arraycopy(orig, 0, copy, 0, orig.length);
+    return copy;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseProcessor.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseProcessor.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseProcessor.java
new file mode 100644
index 0000000..035643d
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TBaseProcessor.java
@@ -0,0 +1,42 @@
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessage;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TMessageType;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType;
+
+public abstract class TBaseProcessor<I> implements TProcessor {
+  private final I iface;
+  private final Map<String,ProcessFunction<I, ? extends TBase>> processMap;
+
+  protected TBaseProcessor(I iface, Map<String, ProcessFunction<I, ? extends TBase>> processFunctionMap) {
+    this.iface = iface;
+    this.processMap = processFunctionMap;
+  }
+
+  public Map<String,ProcessFunction<I, ? extends TBase>> getProcessMapView() {
+    return Collections.unmodifiableMap(processMap);
+  }
+
+  @Override
+  public boolean process(TProtocol in, TProtocol out) throws TException {
+    TMessage msg = in.readMessageBegin();
+    ProcessFunction fn = processMap.get(msg.name);
+    if (fn == null) {
+      TProtocolUtil.skip(in, TType.STRUCT);
+      in.readMessageEnd();
+      TApplicationException x = new TApplicationException(TApplicationException.UNKNOWN_METHOD, "Invalid method name: '"+msg.name+"'");
+      out.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid));
+      x.write(out);
+      out.writeMessageEnd();
+      out.getTransport().flush();
+      return true;
+    }
+    fn.process(msg.seqid, in, out, iface);
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TByteArrayOutputStream.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TByteArrayOutputStream.java
new file mode 100644
index 0000000..f425fd1
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TByteArrayOutputStream.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import java.io.ByteArrayOutputStream;
+
+/**
+ * Class that allows access to the underlying buf without doing deep
+ * copies on it.
+ *
+ */
+public class TByteArrayOutputStream extends ByteArrayOutputStream {
+  public TByteArrayOutputStream(int size) {
+    super(size);
+  }
+
+  public TByteArrayOutputStream() {
+    super();
+  }
+
+  public byte[] get() {
+    return buf;
+  }
+
+  public int len() {
+    return count;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TDeserializer.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TDeserializer.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TDeserializer.java
new file mode 100644
index 0000000..cd81305
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TDeserializer.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TBinaryProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TField;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolFactory;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocolUtil;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TType;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TMemoryInputTransport;
+
+/**
+ * Generic utility for easily deserializing objects from a byte array or Java
+ * String.
+ *
+ */
+public class TDeserializer {
+  private final TProtocol protocol_;
+  private final TMemoryInputTransport trans_;
+
+  /**
+   * Create a new TDeserializer that uses the TBinaryProtocol by default.
+   */
+  public TDeserializer() {
+    this(new TBinaryProtocol.Factory());
+  }
+
+  /**
+   * Create a new TDeserializer. It will use the TProtocol specified by the
+   * factory that is passed in.
+   *
+   * @param protocolFactory Factory to create a protocol
+   */
+  public TDeserializer(TProtocolFactory protocolFactory) {
+    trans_ = new TMemoryInputTransport();
+    protocol_ = protocolFactory.getProtocol(trans_);
+  }
+
+  /**
+   * Deserialize the Thrift object from a byte array.
+   *
+   * @param base The object to read into
+   * @param bytes The array to read from
+   */
+  public void deserialize(TBase base, byte[] bytes) throws TException {
+    try {
+      trans_.reset(bytes);
+      base.read(protocol_);
+    } finally {
+      trans_.clear();
+      protocol_.reset();
+    }
+  }
+
+  /**
+   * Deserialize the Thrift object from a Java string, using a specified
+   * character set for decoding.
+   *
+   * @param base The object to read into
+   * @param data The string to read from
+   * @param charset Valid JVM charset
+   */
+  public void deserialize(TBase base, String data, String charset) throws TException {
+    try {
+      deserialize(base, data.getBytes(charset));
+    } catch (UnsupportedEncodingException uex) {
+      throw new TException("JVM DOES NOT SUPPORT ENCODING: " + charset);
+    } finally {
+      protocol_.reset();
+    }
+  }
+
+  /**
+   * Deserialize only a single Thrift object (addressed by recursively using field id)
+   * from a byte record.   
+   * @param tb The object to read into
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path tb
+   * @param fieldIdPathRest The rest FieldId's that define a path tb
+   * @throws TException 
+   */
+  public void partialDeserialize(TBase tb, byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    try {
+      if (locateField(bytes, fieldIdPathFirst, fieldIdPathRest) != null) {
+        // if this line is reached, iprot will be positioned at the start of tb.
+        tb.read(protocol_);
+      }      
+    } catch (Exception e) {
+      throw new TException(e);
+    } finally {
+      trans_.clear();
+      protocol_.reset();
+    }
+  }
+
+  /**
+   * Deserialize only a boolean field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to a boolean field
+   * @param fieldIdPathRest The rest FieldId's that define a path to a boolean field
+   * @throws TException
+   */
+  public Boolean partialDeserializeBool(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    return (Boolean) partialDeserializeField(TType.BOOL, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only a byte field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to a byte field
+   * @param fieldIdPathRest The rest FieldId's that define a path to a byte field
+   * @throws TException
+   */
+  public Byte partialDeserializeByte(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    return (Byte) partialDeserializeField(TType.BYTE, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only a double field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to a double field
+   * @param fieldIdPathRest The rest FieldId's that define a path to a double field
+   * @throws TException
+   */
+  public Double partialDeserializeDouble(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    return (Double) partialDeserializeField(TType.DOUBLE, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only an i16 field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to an i16 field
+   * @param fieldIdPathRest The rest FieldId's that define a path to an i16 field
+   * @throws TException
+   */
+  public Short partialDeserializeI16(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    return (Short) partialDeserializeField(TType.I16, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only an i32 field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to an i32 field
+   * @param fieldIdPathRest The rest FieldId's that define a path to an i32 field
+   * @throws TException
+   */
+  public Integer partialDeserializeI32(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    return (Integer) partialDeserializeField(TType.I32, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only an i64 field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to an i64 field
+   * @param fieldIdPathRest The rest FieldId's that define a path to an i64 field
+   * @throws TException
+   */
+  public Long partialDeserializeI64(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    return (Long) partialDeserializeField(TType.I64, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only a string field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to a string field
+   * @param fieldIdPathRest The rest FieldId's that define a path to a string field
+   * @throws TException
+   */
+  public String partialDeserializeString(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    return (String) partialDeserializeField(TType.STRING, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only a binary field (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to a binary field
+   * @param fieldIdPathRest The rest FieldId's that define a path to a binary field
+   * @throws TException
+   */
+  public ByteBuffer partialDeserializeByteArray(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    // TType does not have binary, so we use the arbitrary num 100
+    return (ByteBuffer) partialDeserializeField((byte)100, bytes, fieldIdPathFirst, fieldIdPathRest);
+  }
+
+  /**
+   * Deserialize only the id of the field set in a TUnion (addressed by recursively using field id)
+   * from a byte record.
+   * @param bytes The serialized object to read from
+   * @param fieldIdPathFirst First of the FieldId's that define a path to a TUnion
+   * @param fieldIdPathRest The rest FieldId's that define a path to a TUnion
+   * @throws TException
+   */
+  public Short partialDeserializeSetFieldIdInUnion(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest)  throws TException {
+    try {
+      TField field = locateField(bytes, fieldIdPathFirst, fieldIdPathRest);
+      if (field != null){
+        protocol_.readStructBegin(); // The Union
+        return protocol_.readFieldBegin().id; // The field set in the union
+      }
+      return null;
+    } catch (Exception e) {
+      throw new TException(e);
+    } finally {
+      trans_.clear();
+      protocol_.reset();
+    }
+  }
+
+  private Object partialDeserializeField(byte ttype, byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    try {
+      TField field = locateField(bytes, fieldIdPathFirst, fieldIdPathRest);
+      if (field != null) {
+        // if this point is reached, iprot will be positioned at the start of the field.
+        switch(ttype){
+          case TType.BOOL:
+            if (field.type == TType.BOOL){
+              return protocol_.readBool();
+            }
+            break;
+          case TType.BYTE:
+            if (field.type == TType.BYTE) {
+              return protocol_.readByte();
+            }
+            break;
+          case TType.DOUBLE:
+            if (field.type == TType.DOUBLE) {
+              return protocol_.readDouble();
+            }
+            break;
+          case TType.I16:
+            if (field.type == TType.I16) {
+              return protocol_.readI16();
+            }
+            break;
+          case TType.I32:
+            if (field.type == TType.I32) {
+              return protocol_.readI32();
+            }
+            break;
+          case TType.I64:
+            if (field.type == TType.I64) {
+              return protocol_.readI64();
+            }
+            break;
+          case TType.STRING:
+            if (field.type == TType.STRING) {
+              return protocol_.readString();
+            }
+            break;
+          case 100: // hack to differentiate between string and binary
+            if (field.type == TType.STRING) {
+              return protocol_.readBinary();
+            }
+            break;
+        }
+      }
+      return null;
+    } catch (Exception e) {
+      throw new TException(e);
+    } finally {
+      trans_.clear();
+      protocol_.reset();
+    }
+  }
+
+  private TField locateField(byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum ... fieldIdPathRest) throws TException {
+    trans_.reset(bytes);
+
+    TFieldIdEnum[] fieldIdPath= new TFieldIdEnum[fieldIdPathRest.length + 1];
+    fieldIdPath[0] = fieldIdPathFirst;
+    for (int i = 0; i < fieldIdPathRest.length; i++){
+      fieldIdPath[i + 1] = fieldIdPathRest[i];
+    }
+
+    // index into field ID path being currently searched for
+    int curPathIndex = 0;
+
+    // this will be the located field, or null if it is not located
+    TField field = null;
+
+    protocol_.readStructBegin();
+
+    while (curPathIndex < fieldIdPath.length) {
+      field = protocol_.readFieldBegin();
+      // we can stop searching if we either see a stop or we go past the field
+      // id we're looking for (since fields should now be serialized in asc
+      // order).
+      if (field.type == TType.STOP || field.id > fieldIdPath[curPathIndex].getThriftFieldId()) {
+        return null;
+      }
+
+      if (field.id != fieldIdPath[curPathIndex].getThriftFieldId()) {
+        // Not the field we're looking for. Skip field.
+        TProtocolUtil.skip(protocol_, field.type);
+        protocol_.readFieldEnd();
+      } else {
+        // This field is the next step in the path. Step into field.
+        curPathIndex++;
+        if (curPathIndex < fieldIdPath.length) {
+          protocol_.readStructBegin();
+        }
+      }
+    }
+    return field;
+  }
+
+  /**
+   * Deserialize the Thrift object from a Java string, using the default JVM
+   * charset encoding.
+   *
+   * @param base The object to read into
+   * @param data The string to read from
+   */
+  public void fromString(TBase base, String data) throws TException {
+    deserialize(base, data.getBytes());
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnum.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnum.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnum.java
new file mode 100644
index 0000000..01a32de
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnum.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+public interface TEnum {
+  public int getValue();
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnumHelper.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnumHelper.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnumHelper.java
new file mode 100644
index 0000000..8d5b3ff
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TEnumHelper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import java.lang.InstantiationException;
+import java.lang.NoSuchMethodException;
+import java.lang.IllegalAccessException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+/**
+ * Utility class with static methods for interacting with TEnum
+ */
+public class TEnumHelper {
+
+  /**
+   * Given a TEnum class and integer value, this method will return
+   * the associated constant from the given TEnum class.
+   * This method MUST be modified should the name of the 'findByValue' method
+   * change.
+   *
+   * @param enumClass TEnum from which to return a matching constant.
+   * @param value Value for which to return the constant.
+   *
+   * @return The constant in 'enumClass' whose value is 'value' or null if
+   *         something went wrong.
+   */
+  public static TEnum getByValue(Class<? extends TEnum> enumClass, int value) {
+    try {
+      Method method = enumClass.getMethod("findByValue", int.class);
+      return (TEnum) method.invoke(null, value);
+    } catch (NoSuchMethodException nsme) {
+      return null;
+    } catch (IllegalAccessException iae) {
+      return null;
+    } catch (InvocationTargetException ite) {
+      return null;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TException.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TException.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TException.java
new file mode 100644
index 0000000..5317ad9
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+/**
+ * Generic exception class for Thrift.
+ *
+ */
+public class TException extends Exception {
+
+  private static final long serialVersionUID = 1L;
+
+  public TException() {
+    super();
+  }
+
+  public TException(String message) {
+    super(message);
+  }
+
+  public TException(Throwable cause) {
+    super(cause);
+  }
+
+  public TException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldIdEnum.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldIdEnum.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldIdEnum.java
new file mode 100644
index 0000000..8cb4b1b
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldIdEnum.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+/**
+ * Interface for all generated struct Fields objects.
+ */
+public interface TFieldIdEnum {
+  /**
+   * Get the Thrift field id for the named field.
+   */
+  public short getThriftFieldId();
+
+  /**
+   * Get the field's name, exactly as in the IDL.
+   */
+  public String getFieldName();
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldRequirementType.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldRequirementType.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldRequirementType.java
new file mode 100644
index 0000000..8cd29b8
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TFieldRequirementType.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+/**
+ * Requirement type constants.
+ *
+ */
+public final class TFieldRequirementType {
+  public static final byte REQUIRED  = 1;
+  public static final byte OPTIONAL = 2;
+  public static final byte DEFAULT = 3;
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessor.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessor.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessor.java
new file mode 100644
index 0000000..d8d5315
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessor.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
+
+/**
+ * A processor is a generic object which operates upon an input stream and
+ * writes to some output stream.
+ *
+ */
+public interface TProcessor {
+  public boolean process(TProtocol in, TProtocol out)
+    throws TException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessorFactory.java
----------------------------------------------------------------------
diff --git a/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessorFactory.java b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessorFactory.java
new file mode 100644
index 0000000..3747971
--- /dev/null
+++ b/blur-thrift/src/main/java/org/apache/blur/thirdparty/thrift_0_9_0/TProcessorFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.blur.thirdparty.thrift_0_9_0;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransport;
+
+/**
+ * The default processor factory just returns a singleton
+ * instance.
+ */
+public class TProcessorFactory {
+
+  private final TProcessor processor_;
+
+  public TProcessorFactory(TProcessor processor) {
+    processor_ = processor;
+  }
+
+  public TProcessor getProcessor(TTransport trans) {
+    return processor_;
+  }
+}


Mime
View raw message