incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [7/51] [partial] Initial repackage to org.apache.
Date Mon, 03 Sep 2012 03:17:10 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/BlurUtil.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/BlurUtil.java
deleted file mode 100644
index 1ce3a2d..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/BlurUtil.java
+++ /dev/null
@@ -1,762 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static com.nearinfinity.blur.utils.BlurConstants.SHARD_PREFIX;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLongArray;
-
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.SegmentReader;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
-import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.PackedInts.Reader;
-import org.apache.thrift.TBase;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TJSONProtocol;
-import org.apache.thrift.transport.TMemoryBuffer;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.Code;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-
-import com.nearinfinity.blur.log.Log;
-import com.nearinfinity.blur.log.LogFactory;
-import com.nearinfinity.blur.manager.clusterstatus.ZookeeperPathConstants;
-import com.nearinfinity.blur.manager.results.BlurResultIterable;
-import com.nearinfinity.blur.metrics.BlurMetrics;
-import com.nearinfinity.blur.metrics.BlurMetrics.MethodCall;
-import com.nearinfinity.blur.thrift.BException;
-import com.nearinfinity.blur.thrift.generated.Blur.Iface;
-import com.nearinfinity.blur.thrift.generated.BlurQuery;
-import com.nearinfinity.blur.thrift.generated.BlurResult;
-import com.nearinfinity.blur.thrift.generated.BlurResults;
-import com.nearinfinity.blur.thrift.generated.Column;
-import com.nearinfinity.blur.thrift.generated.FetchResult;
-import com.nearinfinity.blur.thrift.generated.Record;
-import com.nearinfinity.blur.thrift.generated.RecordMutation;
-import com.nearinfinity.blur.thrift.generated.RecordMutationType;
-import com.nearinfinity.blur.thrift.generated.Row;
-import com.nearinfinity.blur.thrift.generated.RowMutation;
-import com.nearinfinity.blur.thrift.generated.RowMutationType;
-import com.nearinfinity.blur.thrift.generated.Selector;
-import com.nearinfinity.blur.thrift.generated.SimpleQuery;
-
-public class BlurUtil {
-
-  private static final Object[] EMPTY_OBJECT_ARRAY = new Object[] {};
-  private static final Class<?>[] EMPTY_PARAMETER_TYPES = new Class[] {};
-  private static final Log LOG = LogFactory.getLog(BlurUtil.class);
-  private static final String UNKNOWN = "UNKNOWN";
-
-  @SuppressWarnings("unchecked")
-  public static <T extends Iface> T recordMethodCallsAndAverageTimes(final BlurMetrics metrics, final T t, Class<T> clazz) {
-    InvocationHandler handler = new InvocationHandler() {
-      @Override
-      public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
-        long start = System.nanoTime();
-        try {
-          return method.invoke(t, args);
-        } catch (InvocationTargetException e) {
-          throw e.getTargetException();
-        } finally {
-          long end = System.nanoTime();
-          MethodCall methodCall = metrics.methodCalls.get(method.getName());
-          if (methodCall == null) {
-            methodCall = new MethodCall();
-            metrics.methodCalls.put(method.getName(), methodCall);
-          }
-          methodCall.invokes.incrementAndGet();
-          methodCall.times.addAndGet(end - start);
-        }
-      }
-    };
-    return (T) Proxy.newProxyInstance(clazz.getClassLoader(), new Class[] { clazz }, handler);
-  }
-
-  public static void setupZookeeper(ZooKeeper zookeeper) throws KeeperException, InterruptedException {
-    setupZookeeper(zookeeper, null);
-  }
-
-  public synchronized static void setupZookeeper(ZooKeeper zookeeper, String cluster) throws KeeperException, InterruptedException {
-    BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getBasePath());
-    BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getOnlineControllersPath());
-    BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getClustersPath());
-    if (cluster != null) {
-      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getClusterPath(cluster));
-      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getSafemodePath(cluster));
-      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getRegisteredShardsPath(cluster));
-      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getOnlinePath(cluster));
-      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getOnlineShardsPath(cluster));
-      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getTablesPath(cluster));
-    }
-  }
-
-  public static BlurQuery newSimpleQuery(String query) {
-    BlurQuery blurQuery = new BlurQuery();
-    SimpleQuery simpleQuery = new SimpleQuery();
-    simpleQuery.setQueryStr(query);
-    blurQuery.setSimpleQuery(simpleQuery);
-    blurQuery.setSelector(new Selector());
-    return blurQuery;
-  }
-
-  public static void createIfMissing(ZooKeeper zookeeper, String path) throws KeeperException, InterruptedException {
-    if (zookeeper.exists(path, false) == null) {
-      try {
-        zookeeper.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-      } catch (KeeperException e) {
-        if (e.code() == Code.NODEEXISTS) {
-          return;
-        }
-        throw e;
-      }
-    }
-  }
-
-  public static List<Long> getList(AtomicLongArray atomicLongArray) {
-    if (atomicLongArray == null) {
-      return null;
-    }
-    List<Long> counts = new ArrayList<Long>(atomicLongArray.length());
-    for (int i = 0; i < atomicLongArray.length(); i++) {
-      counts.add(atomicLongArray.get(i));
-    }
-    return counts;
-  }
-
-  public static void quietClose(Object... close) {
-    if (close == null) {
-      return;
-    }
-    for (Object object : close) {
-      if (object != null) {
-        close(object);
-      }
-    }
-  }
-
-  private static void close(Object object) {
-    Class<? extends Object> clazz = object.getClass();
-    Method method;
-    try {
-      method = clazz.getMethod("close", EMPTY_PARAMETER_TYPES);
-    } catch (SecurityException e) {
-      throw new RuntimeException(e);
-    } catch (NoSuchMethodException e) {
-      return;
-    }
-    try {
-      method.invoke(object, EMPTY_OBJECT_ARRAY);
-    } catch (Exception e) {
-      LOG.error("Error while trying to close object [{0}]", e, object);
-    }
-  }
-
-  public static Selector newSelector(String locationId) {
-    Selector selector = new Selector();
-    selector.locationId = locationId;
-    return selector;
-  }
-
-  public static RecordMutation newRecordMutation(String family, String recordId, Column... columns) {
-    return newRecordMutation(RecordMutationType.REPLACE_ENTIRE_RECORD, family, recordId, columns);
-  }
-
-  public static RecordMutation newRecordMutation(RecordMutationType type, String family, String recordId, Column... columns) {
-    Record record = new Record();
-    record.setRecordId(recordId);
-    record.setFamily(family);
-    for (Column column : columns) {
-      record.addToColumns(column);
-    }
-
-    RecordMutation mutation = new RecordMutation();
-    mutation.setRecordMutationType(type);
-    mutation.setRecord(record);
-    return mutation;
-  }
-
-  public static RecordMutation findRecordMutation(RowMutation mutation, Record record) {
-    for (RecordMutation recordMutation : mutation.recordMutations) {
-      if (match(recordMutation, record)) {
-        return recordMutation;
-      }
-    }
-    return null;
-  }
-
-  public static boolean match(RecordMutation mutation, Record record) {
-    return match(mutation.record, record);
-  }
-
-  public static boolean match(Record left, Record right) {
-    return left.recordId.equals(right.recordId) && left.family.equals(right.family);
-  }
-
-  public static RowMutation newRowMutation(String table, String rowId, RecordMutation... mutations) {
-    return newRowMutation(RowMutationType.REPLACE_ROW, table, rowId, mutations);
-  }
-
-  public static RowMutation newRowMutation(RowMutationType type, String table, String rowId, RecordMutation... mutations) {
-    RowMutation mutation = new RowMutation();
-    mutation.setRowId(rowId);
-    mutation.setTable(table);
-    mutation.setRowMutationType(type);
-    for (RecordMutation recordMutation : mutations) {
-      mutation.addToRecordMutations(recordMutation);
-    }
-    return mutation;
-  }
-
-  public static Record newRecord(String family, String recordId, Column... columns) {
-    Record record = new Record();
-    record.setRecordId(recordId);
-    record.setFamily(family);
-    record.setColumns(Arrays.asList(columns));
-    return record;
-  }
-
-  public static Row newRow(String rowId, Record... records) {
-    Row row = new Row().setId(rowId);
-    for (Record record : records) {
-      row.addToRecords(record);
-    }
-    return row;
-  }
-
-  public static Column newColumn(String name, String value) {
-    return new Column().setName(name).setValue(value);
-  }
-
-  public static byte[] toBytes(Serializable serializable) {
-    try {
-      ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-      ObjectOutputStream stream = new ObjectOutputStream(outputStream);
-      stream.writeObject(serializable);
-      stream.close();
-      return outputStream.toByteArray();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  public static Serializable fromBytes(byte[] bs) {
-    ObjectInputStream stream = null;
-    try {
-      stream = new ObjectInputStream(new ByteArrayInputStream(bs));
-      return (Serializable) stream.readObject();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    } finally {
-      if (stream != null) {
-        try {
-          stream.close();
-        } catch (IOException e) {
-          // eat
-        }
-      }
-    }
-  }
-
-  public static List<Long> toList(AtomicLongArray atomicLongArray) {
-    if (atomicLongArray == null) {
-      return null;
-    }
-    int length = atomicLongArray.length();
-    List<Long> result = new ArrayList<Long>(length);
-    for (int i = 0; i < length; i++) {
-      result.add(atomicLongArray.get(i));
-    }
-    return result;
-  }
-
-  public static AtomicLongArray getAtomicLongArraySameLengthAsList(List<?> list) {
-    if (list == null) {
-      return null;
-    }
-    return new AtomicLongArray(list.size());
-  }
-
-  public static BlurResults convertToHits(BlurResultIterable hitsIterable, BlurQuery query, AtomicLongArray facetCounts, ExecutorService executor, Selector selector,
-      final Iface iface, final String table) throws InterruptedException, ExecutionException {
-    BlurResults results = new BlurResults();
-    results.setTotalResults(hitsIterable.getTotalResults());
-    results.setShardInfo(hitsIterable.getShardInfo());
-    if (query.minimumNumberOfResults > 0) {
-      hitsIterable.skipTo(query.start);
-      int count = 0;
-      Iterator<BlurResult> iterator = hitsIterable.iterator();
-      while (iterator.hasNext() && count < query.fetch) {
-        results.addToResults(iterator.next());
-        count++;
-      }
-    }
-    if (results.results == null) {
-      results.results = new ArrayList<BlurResult>();
-    }
-    if (facetCounts != null) {
-      results.facetCounts = BlurUtil.toList(facetCounts);
-    }
-    if (selector != null) {
-      List<Future<FetchResult>> futures = new ArrayList<Future<FetchResult>>();
-      for (int i = 0; i < results.results.size(); i++) {
-        BlurResult result = results.results.get(i);
-        final Selector s = new Selector(selector);
-        s.setLocationId(result.locationId);
-        futures.add(executor.submit(new Callable<FetchResult>() {
-          @Override
-          public FetchResult call() throws Exception {
-            return iface.fetchRow(table, s);
-          }
-        }));
-      }
-      for (int i = 0; i < results.results.size(); i++) {
-        Future<FetchResult> future = futures.get(i);
-        BlurResult result = results.results.get(i);
-        result.setFetchResult(future.get());
-      }
-    }
-    results.query = query;
-    results.query.selector = selector;
-    return results;
-  }
-
-  public static Query readQuery(byte[] bs) throws BException {
-    return readObject(bs);
-  }
-
-  public static byte[] writeQuery(Query query) throws BException {
-    return writeObject(query);
-  }
-
-  public static Sort readSort(byte[] bs) throws BException {
-    return readObject(bs);
-  }
-
-  public static byte[] writeSort(Sort sort) throws BException {
-    return writeObject(sort);
-  }
-
-  public static Filter readFilter(byte[] bs) throws BException {
-    return readObject(bs);
-  }
-
-  public static byte[] writeFilter(Filter filter) throws BException {
-    return writeObject(filter);
-  }
-
-  private static byte[] writeObject(Serializable o) throws BException {
-    if (o == null) {
-      return null;
-    }
-    try {
-      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-      ObjectOutputStream outputStream = new ObjectOutputStream(byteArrayOutputStream);
-      outputStream.writeObject(o);
-      outputStream.close();
-      return byteArrayOutputStream.toByteArray();
-    } catch (IOException e) {
-      throw new BException("Unknown error", e);
-    }
-  }
-
-  @SuppressWarnings("unchecked")
-  private static <T> T readObject(byte[] bs) throws BException {
-    if (bs == null) {
-      return null;
-    }
-    ObjectInputStream inputStream = null;
-    try {
-      inputStream = new ObjectInputStream(new ByteArrayInputStream(bs));
-      return (T) inputStream.readObject();
-    } catch (IOException e) {
-      throw new BException("Unknown error", e);
-    } catch (ClassNotFoundException e) {
-      throw new BException("Unknown error", e);
-    } finally {
-      if (inputStream != null) {
-        try {
-          inputStream.close();
-        } catch (IOException e) {
-          throw new BException("Unknown error", e);
-        }
-      }
-    }
-  }
-
-  public static void setStartTime(BlurQuery query) {
-    if (query.startTime == 0) {
-      query.startTime = System.currentTimeMillis();
-    }
-  }
-
-  public static String getVersion() {
-    String path = "/META-INF/maven/com.nearinfinity.blur/blur-core/pom.properties";
-    InputStream inputStream = BlurUtil.class.getResourceAsStream(path);
-    if (inputStream == null) {
-      return UNKNOWN;
-    }
-    Properties prop = new Properties();
-    try {
-      prop.load(inputStream);
-    } catch (IOException e) {
-      LOG.error("Unknown error while getting version.", e);
-      return UNKNOWN;
-    }
-    Object verison = prop.get("version");
-    if (verison == null) {
-      return UNKNOWN;
-    }
-    return verison.toString();
-  }
-
-  public static void unlockForSafeMode(ZooKeeper zookeeper, String lockPath) throws InterruptedException, KeeperException {
-    zookeeper.delete(lockPath, -1);
-    LOG.info("Lock released.");
-  }
-
-  public static String lockForSafeMode(ZooKeeper zookeeper, String nodeName, String cluster) throws KeeperException, InterruptedException {
-    LOG.info("Getting safe mode lock.");
-    final Object lock = new Object();
-    String blurSafemodePath = ZookeeperPathConstants.getSafemodePath(cluster);
-    String newPath = zookeeper.create(blurSafemodePath + "/safemode-", nodeName.getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
-    Watcher watcher = new Watcher() {
-      @Override
-      public void process(WatchedEvent event) {
-        synchronized (lock) {
-          lock.notifyAll();
-        }
-      }
-    };
-    while (true) {
-      synchronized (lock) {
-        List<String> children = new ArrayList<String>(zookeeper.getChildren(blurSafemodePath, watcher));
-        Collections.sort(children);
-        if (newPath.equals(blurSafemodePath + "/" + children.get(0))) {
-          LOG.info("Lock aquired.");
-          return newPath;
-        } else {
-          lock.wait(BlurConstants.ZK_WAIT_TIME);
-        }
-      }
-    }
-  }
-
-  public static String getShardName(String prefix, int id) {
-    return prefix + buffer(id, 8);
-  }
-
-  private static String buffer(int value, int length) {
-    String str = Integer.toString(value);
-    while (str.length() < length) {
-      str = "0" + str;
-    }
-    return str;
-  }
-
-  public static String humanizeTime(long time, TimeUnit unit) {
-    long seconds = unit.toSeconds(time);
-    long hours = getHours(seconds);
-    seconds = seconds - TimeUnit.HOURS.toSeconds(hours);
-    long minutes = getMinutes(seconds);
-    seconds = seconds - TimeUnit.MINUTES.toSeconds(minutes);
-    return humanizeTime(hours, minutes, seconds);
-  }
-
-  public static String humanizeTime(long hours, long minutes, long seconds) {
-    StringBuilder builder = new StringBuilder();
-    if (hours == 0 && minutes != 0) {
-      addMinutes(builder, minutes);
-    } else if (hours != 0) {
-      addHours(builder, hours);
-      addMinutes(builder, minutes);
-    }
-    addSeconds(builder, seconds);
-    return builder.toString().trim();
-  }
-
-  private static void addHours(StringBuilder builder, long hours) {
-    builder.append(hours).append(" hours ");
-  }
-
-  private static void addMinutes(StringBuilder builder, long minutes) {
-    builder.append(minutes).append(" minutes ");
-  }
-
-  private static void addSeconds(StringBuilder builder, long seconds) {
-    builder.append(seconds).append(" seconds ");
-  }
-
-  private static long getMinutes(long seconds) {
-    return seconds / TimeUnit.MINUTES.toSeconds(1);
-  }
-
-  private static long getHours(long seconds) {
-    return seconds / TimeUnit.HOURS.toSeconds(1);
-  }
-
-  @SuppressWarnings("unchecked")
-  public static long getMemoryUsage(IndexReader r) {
-    try {
-      if (r instanceof SegmentReader) {
-        long size = 0;
-        SegmentReader segmentReader = (SegmentReader) r;
-        Object segmentCoreReaders = getSegmentCoreReaders(segmentReader);
-        Object termInfosReader = getTermInfosReader(segmentCoreReaders);
-        Object termInfosReaderIndex = getTermInfosReaderIndex(termInfosReader);
-        PagedBytesDataInput dataInput = getDataInput(termInfosReaderIndex);
-        PackedInts.Reader indexToDataOffset = getIndexToDataOffset(termInfosReaderIndex);
-
-        Object pagedBytes = BlurUtil.getField("this$0", dataInput);
-        List<byte[]> blocks = (List<byte[]>) BlurUtil.getField("blocks", pagedBytes);
-        for (byte[] block : blocks) {
-          size += block.length;
-        }
-
-        try {
-          Class<? extends Reader> clazz = indexToDataOffset.getClass();
-          Method method = clazz.getMethod("ramBytesUsed", EMPTY_PARAMETER_TYPES);
-          method.setAccessible(true);
-          Long ramBytesUsed = (Long) method.invoke(indexToDataOffset, EMPTY_OBJECT_ARRAY);
-          size += ramBytesUsed;
-        } catch (Exception e) {
-          throw new RuntimeException(e);
-        }
-        return size;
-      }
-      IndexReader[] readers = r.getSequentialSubReaders();
-      long total = 0;
-      if (readers != null) {
-        for (IndexReader reader : readers) {
-          total += getMemoryUsage(reader);
-        }
-      }
-      return total;
-    } catch (Exception e) {
-      LOG.error("Unknown error during getMemoryUsage call", e);
-      return 0;
-    }
-  }
-
-  private static PackedInts.Reader getIndexToDataOffset(Object termInfosReaderIndex) {
-    return (Reader) getField("indexToDataOffset", termInfosReaderIndex);
-  }
-
-  private static PagedBytesDataInput getDataInput(Object termInfosReaderIndex) {
-    return (PagedBytesDataInput) getField("dataInput", termInfosReaderIndex);
-  }
-
-  private static Object getTermInfosReaderIndex(Object termInfosReader) {
-    return getField("index", termInfosReader);
-  }
-
-  private static Object getTermInfosReader(Object segmentCoreReaders) {
-    return getField("tis", segmentCoreReaders);
-  }
-
-  private static Object getSegmentCoreReaders(SegmentReader segmentReader) {
-    return getField("core", segmentReader, SegmentReader.class);
-  }
-
-  private static Object getField(String name, Object o) {
-    Class<? extends Object> clazz = o.getClass();
-    return getField(name, o, clazz);
-  }
-
-  private static Object getField(String name, Object o, Class<? extends Object> clazz) {
-    try {
-      Field field = clazz.getDeclaredField(name);
-      field.setAccessible(true);
-      return field.get(o);
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  public static void createPath(ZooKeeper zookeeper, String path, byte[] data) throws KeeperException, InterruptedException {
-    zookeeper.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-  }
-
-  public static void setupFileSystem(String uri, int shardCount) throws IOException {
-    Path tablePath = new Path(uri);
-    FileSystem fileSystem = FileSystem.get(tablePath.toUri(), new Configuration());
-    if (createPath(fileSystem, tablePath)) {
-      LOG.info("Table uri existed.");
-      validateShardCount(shardCount, fileSystem, tablePath);
-    }
-    for (int i = 0; i < shardCount; i++) {
-      String shardName = BlurUtil.getShardName(SHARD_PREFIX, i);
-      Path shardPath = new Path(tablePath, shardName);
-      createPath(fileSystem, shardPath);
-    }
-  }
-
-  public static void validateShardCount(int shardCount, FileSystem fileSystem, Path tablePath) throws IOException {
-    FileStatus[] listStatus = fileSystem.listStatus(tablePath);
-    if (listStatus.length != shardCount) {
-      LOG.error("Number of directories in table path [" + tablePath + "] does not match definition of [" + shardCount + "] shard count.");
-      throw new RuntimeException("Number of directories in table path [" + tablePath + "] does not match definition of [" + shardCount + "] shard count.");
-    }
-  }
-
-  public static boolean createPath(FileSystem fileSystem, Path path) throws IOException {
-    if (!fileSystem.exists(path)) {
-      LOG.info("Path [{0}] does not exist, creating.", path);
-      fileSystem.mkdirs(path);
-      return false;
-    }
-    return true;
-  }
-
-  public static int zeroCheck(int i, String message) {
-    if (i < 1) {
-      throw new RuntimeException(message);
-    }
-    return i;
-  }
-
-  public static <T> T nullCheck(T t, String message) {
-    if (t == null) {
-      throw new NullPointerException(message);
-    }
-    return t;
-  }
-
-  @SuppressWarnings("unchecked")
-  public static <T> T getInstance(String className, Class<T> c) {
-    Class<?> clazz;
-    try {
-      clazz = Class.forName(className);
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException(e);
-    }
-    try {
-      return (T) configure(clazz.newInstance());
-    } catch (InstantiationException e) {
-      throw new RuntimeException(e);
-    } catch (IllegalAccessException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  public static <T> T configure(T t) {
-    if (t instanceof Configurable) {
-      Configurable configurable = (Configurable) t;
-      configurable.setConf(new Configuration());
-    }
-    return t;
-  }
-
-  public static byte[] read(TBase<?, ?> base) {
-    if (base == null) {
-      return null;
-    }
-    TMemoryBuffer trans = new TMemoryBuffer(1024);
-    TJSONProtocol protocol = new TJSONProtocol(trans);
-    try {
-      base.write(protocol);
-    } catch (TException e) {
-      throw new RuntimeException(e);
-    }
-    trans.close();
-    byte[] buf = new byte[trans.length()];
-    System.arraycopy(trans.getArray(), 0, buf, 0, trans.length());
-    return buf;
-  }
-
-  public static void write(byte[] data, TBase<?, ?> base) {
-    nullCheck(null, "Data cannot be null.");
-    TMemoryBuffer trans = new TMemoryBuffer(1024);
-    TJSONProtocol protocol = new TJSONProtocol(trans);
-    try {
-      trans.write(data);
-      base.read(protocol);
-    } catch (TException e) {
-      throw new RuntimeException(e);
-    }
-    trans.close();
-  }
-
-  public static void removeAll(ZooKeeper zooKeeper, String path) throws KeeperException, InterruptedException {
-    List<String> list = zooKeeper.getChildren(path, false);
-    for (String p : list) {
-      removeAll(zooKeeper, path + "/" + p);
-    }
-    LOG.info("Removing path [{0}]", path);
-    zooKeeper.delete(path, -1);
-  }
-
-  public static void removeIndexFiles(String uri) throws IOException {
-    Path tablePath = new Path(uri);
-    FileSystem fileSystem = FileSystem.get(tablePath.toUri(), new Configuration());
-    fileSystem.delete(tablePath, true);
-  }
-
-  public static RowMutation toRowMutation(String table, Row row) {
-    RowMutation rowMutation = new RowMutation();
-    rowMutation.setRowId(row.getId());
-    rowMutation.setTable(table);
-    rowMutation.setRowMutationType(RowMutationType.REPLACE_ROW);
-    List<Record> records = row.getRecords();
-    for (Record record : records) {
-      rowMutation.addToRecordMutations(toRecordMutation(record));
-    }
-    return rowMutation;
-  }
-
-  public static RecordMutation toRecordMutation(Record record) {
-    RecordMutation recordMutation = new RecordMutation();
-    recordMutation.setRecord(record);
-    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
-    return recordMutation;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/Converter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/Converter.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/Converter.java
deleted file mode 100644
index 66339c1..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/Converter.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-public interface Converter<F, T> {
-  T convert(F from) throws Exception;
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ForkJoin.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ForkJoin.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ForkJoin.java
deleted file mode 100644
index b87f6a9..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ForkJoin.java
+++ /dev/null
@@ -1,91 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-import com.nearinfinity.blur.log.Log;
-import com.nearinfinity.blur.log.LogFactory;
-import com.nearinfinity.blur.thrift.generated.BlurException;
-import com.nearinfinity.blur.utils.BlurExecutorCompletionService.Cancel;
-
-public class ForkJoin {
-
-  private static Log LOG = LogFactory.getLog(ForkJoin.class);
-
-  public static interface ParallelCall<INPUT, OUTPUT> {
-    OUTPUT call(INPUT input) throws Exception;
-  }
-
-  public static interface ParallelReturn<OUTPUT> {
-    OUTPUT merge(Merger<OUTPUT> merger) throws BlurException;
-  }
-
-  public static interface Merger<OUTPUT> {
-    OUTPUT merge(BlurExecutorCompletionService<OUTPUT> service) throws BlurException;
-  }
-
-  public static Cancel CANCEL = new Cancel() {
-    @Override
-    public void cancel() {
-      // do nothing
-    }
-  };
-
-  public static <INPUT, OUTPUT> ParallelReturn<OUTPUT> execute(ExecutorService executor, Iterable<INPUT> it, final ParallelCall<INPUT, OUTPUT> parallelCall) {
-    return execute(executor, it, parallelCall, CANCEL);
-  }
-
-  public static <INPUT, OUTPUT> ParallelReturn<OUTPUT> execute(ExecutorService executor, Iterable<INPUT> it, final ParallelCall<INPUT, OUTPUT> parallelCall, Cancel cancel) {
-    final BlurExecutorCompletionService<OUTPUT> service = new BlurExecutorCompletionService<OUTPUT>(executor, cancel);
-    for (final INPUT input : it) {
-      service.submit(new Callable<OUTPUT>() {
-        @Override
-        public OUTPUT call() throws Exception {
-          return parallelCall.call(input);
-        }
-      });
-    }
-    return new ParallelReturn<OUTPUT>() {
-      @Override
-      public OUTPUT merge(Merger<OUTPUT> merger) throws BlurException {
-        boolean exception = true;
-        try {
-          OUTPUT merge = merger.merge(service);
-          exception = false;
-          return merge;
-        } finally {
-          if (exception) {
-            service.cancelAll();
-          }
-        }
-      }
-    };
-  }
-
-  public static <T> T ignoreExecutionException(Future<T> future, T defaultValue) throws InterruptedException {
-    try {
-      return future.get();
-    } catch (ExecutionException e) {
-      LOG.error("Error while trying to execute task [{0}]", e, e.getMessage());
-      return defaultValue;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IterableConverter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IterableConverter.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IterableConverter.java
deleted file mode 100644
index 6a57242..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IterableConverter.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.Iterator;
-
-public class IterableConverter<F, T> implements Iterable<T> {
-
-  private Converter<F, T> converter;
-  private Iterable<F> iterable;
-
-  public IterableConverter(Iterable<F> iterable, Converter<F, T> converter) {
-    this.converter = converter;
-    this.iterable = iterable;
-  }
-
-  @Override
-  public Iterator<T> iterator() {
-    return new IteratorConverter<F, T>(iterable.iterator(), converter);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IteratorConverter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IteratorConverter.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IteratorConverter.java
deleted file mode 100644
index 93f9012..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/IteratorConverter.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.Iterator;
-
-public class IteratorConverter<F, T> implements Iterator<T> {
-
-  private Converter<F, T> converter;
-  private Iterator<F> iterator;
-
-  public IteratorConverter(Iterator<F> iterator, Converter<F, T> converter) {
-    this.converter = converter;
-    this.iterator = iterator;
-  }
-
-  @Override
-  public boolean hasNext() {
-    return iterator.hasNext();
-  }
-
-  @Override
-  public T next() {
-    try {
-      return converter.convert(iterator.next());
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public void remove() {
-    iterator.remove();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ObjectSize.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ObjectSize.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ObjectSize.java
deleted file mode 100644
index 0cf854b..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ObjectSize.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.lang.instrument.Instrumentation;
-
-public class ObjectSize {
-
-  private static Instrumentation instrumentation;
-
-  public static void premain(String agentArgs, Instrumentation inst) {
-    instrumentation = inst;
-  }
-
-  public static long getSizeInBytes(Object o) {
-    if (instrumentation == null)
-      return -1;
-    if (o == null)
-      return 0;
-    return instrumentation.getObjectSize(o);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/PrimeDocCache.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/PrimeDocCache.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/PrimeDocCache.java
deleted file mode 100644
index 187830c..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/PrimeDocCache.java
+++ /dev/null
@@ -1,68 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexReader.ReaderClosedListener;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.util.OpenBitSet;
-
-import com.nearinfinity.blur.log.Log;
-import com.nearinfinity.blur.log.LogFactory;
-
-public class PrimeDocCache {
-
-  private static final Log LOG = LogFactory.getLog(PrimeDocCache.class);
-
-  public static final OpenBitSet EMPTY_BIT_SET = new OpenBitSet();
-
-  private static Map<Object, OpenBitSet> primeDocMap = new ConcurrentHashMap<Object, OpenBitSet>();
-
-  /**
-   * The way this method is called via warm up methods the likelihood of
-   * creating multiple bitsets during a race condition is very low, that's why
-   * this method is not synced.
-   */
-  public static OpenBitSet getPrimeDocBitSet(IndexReader reader) throws IOException {
-    Object key = reader.getCoreCacheKey();
-    OpenBitSet bitSet = primeDocMap.get(key);
-    if (bitSet == null) {
-      reader.addReaderClosedListener(new ReaderClosedListener() {
-        @Override
-        public void onClose(IndexReader reader) {
-          Object key = reader.getCoreCacheKey();
-          LOG.debug("Current size [" + primeDocMap.size() + "] Prime Doc BitSet removing for segment [" + reader + "]");
-          primeDocMap.remove(key);
-        }
-      });
-      LOG.debug("Prime Doc BitSet missing for segment [" + reader + "] current size [" + primeDocMap.size() + "]");
-      bitSet = new OpenBitSet(reader.maxDoc());
-      primeDocMap.put(key, bitSet);
-      TermDocs termDocs = reader.termDocs(BlurConstants.PRIME_DOC_TERM);
-      while (termDocs.next()) {
-        bitSet.set(termDocs.doc());
-      }
-      termDocs.close();
-    }
-    return bitSet;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCache.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCache.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCache.java
deleted file mode 100644
index b5fc053..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCache.java
+++ /dev/null
@@ -1,106 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.SortedSet;
-
-import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.googlecode.concurrentlinkedhashmap.EvictionListener;
-import com.nearinfinity.blur.log.Log;
-import com.nearinfinity.blur.log.LogFactory;
-import com.nearinfinity.blur.thrift.generated.BlurQuery;
-import com.nearinfinity.blur.thrift.generated.BlurResults;
-
-public class QueryCache implements EvictionListener<QueryCacheKey, QueryCacheEntry> {
-
-  private static final Log LOG = LogFactory.getLog(QueryCache.class);
-
-  private long _ttl;
-  private String _name;
-  private int _cachedElements;
-  private ConcurrentLinkedHashMap<QueryCacheKey, QueryCacheEntry> _cache;
-
-  public QueryCache(String name, int cachedElements, long ttl) {
-    _name = name;
-    _cachedElements = cachedElements;
-    _ttl = ttl;
-    _cache = new ConcurrentLinkedHashMap.Builder<QueryCacheKey, QueryCacheEntry>().maximumWeightedCapacity(_cachedElements).listener(this).build();
-  }
-
-  @Override
-  public void onEviction(QueryCacheKey key, QueryCacheEntry value) {
-    LOG.debug("Cache [" + _name + "] key [" + key + "] value [" + value + "] evicted.");
-  }
-
-  @SuppressWarnings("deprecation")
-  public static QueryCacheKey getNormalizedBlurQueryKey(String table, BlurQuery blurQuery) {
-    BlurQuery newBlurQuery = new BlurQuery(blurQuery);
-    newBlurQuery.allowStaleData = false;
-    newBlurQuery.useCacheIfPresent = false;
-    newBlurQuery.userContext = null;
-    newBlurQuery.maxQueryTime = 0;
-    newBlurQuery.uuid = 0;
-    newBlurQuery.startTime = 0;
-    newBlurQuery.modifyFileCaches = false;
-    return new QueryCacheKey(table, newBlurQuery);
-  }
-
-  public boolean isValid(QueryCacheEntry entry, SortedSet<String> currentShards) {
-    if (!isValid(entry)) {
-      return false;
-    }
-    if (!entry.shards.equals(currentShards)) {
-      return false;
-    }
-    return true;
-  }
-
-  public boolean isValid(QueryCacheEntry entry) {
-    if (entry == null) {
-      return false;
-    }
-    if (entry.timestamp + _ttl < System.currentTimeMillis()) {
-      return false;
-    }
-    return true;
-  }
-
-  public BlurResults cache(String table, BlurQuery original, BlurResults results) {
-    if (results == null) {
-      return null;
-    }
-    if (original != null && original.cacheResult) {
-      LOG.debug("Caching results for query [{0}]", original);
-      BlurResults cacheResults = new BlurResults(results);
-      cacheResults.query = null;
-      put(getNormalizedBlurQueryKey(table, original), new QueryCacheEntry(cacheResults));
-    }
-    return results;
-  }
-
-  public void put(QueryCacheKey key, QueryCacheEntry value) {
-    _cache.put(key, value);
-  }
-
-  public QueryCacheEntry get(QueryCacheKey key) {
-    return _cache.get(key);
-  }
-
-  public void remove(QueryCacheKey key) {
-    _cache.remove(key);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheEntry.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheEntry.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheEntry.java
deleted file mode 100644
index 3fae7e8..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheEntry.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import com.nearinfinity.blur.thrift.generated.BlurQuery;
-import com.nearinfinity.blur.thrift.generated.BlurResults;
-
-public class QueryCacheEntry {
-  public BlurResults results;
-  public long timestamp;
-  public SortedSet<String> shards;
-
-  public QueryCacheEntry(BlurResults cacheResults) {
-    results = cacheResults;
-    timestamp = System.currentTimeMillis();
-    if (cacheResults != null && cacheResults.shardInfo != null) {
-      shards = new TreeSet<String>(cacheResults.shardInfo.keySet());
-    } else {
-      shards = new TreeSet<String>();
-    }
-  }
-
-  public BlurResults getBlurResults(BlurQuery blurQuery) {
-    BlurResults blurResults = new BlurResults(results);
-    blurResults.query = blurQuery;
-    return blurResults;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheKey.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheKey.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheKey.java
deleted file mode 100644
index cf1ffa7..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/QueryCacheKey.java
+++ /dev/null
@@ -1,77 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import com.nearinfinity.blur.thrift.generated.BlurQuery;
-
-public class QueryCacheKey {
-  private String table;
-  private BlurQuery query;
-
-  public String getTable() {
-    return table;
-  }
-
-  public void setTable(String table) {
-    this.table = table;
-  }
-
-  public BlurQuery getQuery() {
-    return query;
-  }
-
-  public void setQuery(BlurQuery query) {
-    this.query = query;
-  }
-
-  public QueryCacheKey(String table, BlurQuery query) {
-    this.table = table;
-    this.query = query;
-  }
-
-  @Override
-  public int hashCode() {
-    final int prime = 31;
-    int result = 1;
-    result = prime * result + ((query == null) ? 0 : query.hashCode());
-    result = prime * result + ((table == null) ? 0 : table.hashCode());
-    return result;
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj)
-      return true;
-    if (obj == null)
-      return false;
-    if (getClass() != obj.getClass())
-      return false;
-    QueryCacheKey other = (QueryCacheKey) obj;
-    if (query == null) {
-      if (other.query != null)
-        return false;
-    } else if (!query.equals(other.query))
-      return false;
-    if (table == null) {
-      if (other.table != null)
-        return false;
-    } else if (!table.equals(other.table))
-      return false;
-    return true;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReaderBlurRecord.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReaderBlurRecord.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReaderBlurRecord.java
deleted file mode 100644
index 7e63b37..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReaderBlurRecord.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to You under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-public interface ReaderBlurRecord {
-
-  void setRecordIdStr(String value);
-
-  void addColumn(String name, String value);
-
-  void setFamilyStr(String family);
-
-  void setRowIdStr(String rowId);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReferenceIterable.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReferenceIterable.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReferenceIterable.java
deleted file mode 100644
index d02f0b1..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/ReferenceIterable.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-public class ReferenceIterable<T> implements Iterable<T> {
-
-  private T t;
-
-  public ReferenceIterable(T t) {
-    this.t = t;
-  }
-
-  @Override
-  public Iterator<T> iterator() {
-    return new Iterator<T>() {
-      boolean taken = false;
-
-      @Override
-      public boolean hasNext() {
-        return !taken;
-      }
-
-      @Override
-      public T next() {
-        if (taken) {
-          throw new NoSuchElementException();
-        }
-        taken = true;
-        return t;
-      }
-
-      @Override
-      public void remove() {
-        throw new UnsupportedOperationException();
-      }
-    };
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowDocumentUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowDocumentUtil.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowDocumentUtil.java
deleted file mode 100644
index cdfd016..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowDocumentUtil.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static com.nearinfinity.blur.utils.BlurConstants.RECORD_ID;
-import static com.nearinfinity.blur.utils.BlurConstants.ROW_ID;
-import static com.nearinfinity.blur.utils.BlurConstants.SEP;
-
-import java.util.ArrayList;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
-
-import com.nearinfinity.blur.thrift.generated.FetchRecordResult;
-import com.nearinfinity.blur.thrift.generated.Record;
-import com.nearinfinity.blur.thrift.generated.Row;
-
-public class RowDocumentUtil {
-
-  public static FetchRecordResult getColumns(Document document) {
-    FetchRecordResult result = new FetchRecordResult();
-    BlurThriftRecord record = new BlurThriftRecord();
-    String rowId = readRecord(document, record);
-    result.setRecord(record);
-    result.setRowid(rowId);
-    return result;
-  }
-
-  public static Row getRow(Iterable<Document> docs) {
-    Row row = new Row();
-    boolean empty = true;
-    if (docs == null) {
-      return null;
-    }
-    for (Document document : docs) {
-      empty = false;
-      BlurThriftRecord record = new BlurThriftRecord();
-      String rowId = readRecord(document, record);
-      if (record.getColumns() != null) {
-        row.addToRecords(record);
-      }
-      if (row.id == null) {
-        row.setId(rowId);
-      }
-      row.recordCount++;
-    }
-    if (empty) {
-      return null;
-    }
-    if (row.records == null) {
-      row.records = new ArrayList<Record>();
-    }
-    return row;
-  }
-
-  public static String readRecord(Document document, ReaderBlurRecord reader) {
-    String rowId = null;
-    String family = null;
-    for (Fieldable field : document.getFields()) {
-      if (field.name().equals(ROW_ID)) {
-        rowId = field.stringValue();
-      } else if (field.name().equals(RECORD_ID)) {
-        reader.setRecordIdStr(field.stringValue());
-      } else {
-        String name = field.name();
-        int index = name.indexOf(SEP);
-        if (index < 0) {
-          continue;
-        } else if (family == null) {
-          family = name.substring(0, index);
-        }
-        name = name.substring(index + 1);
-        reader.addColumn(name, field.stringValue());
-      }
-    }
-    reader.setFamilyStr(family);
-    reader.setRowIdStr(rowId);
-    return rowId;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowIndexWriter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowIndexWriter.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowIndexWriter.java
deleted file mode 100644
index 79d7309..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/RowIndexWriter.java
+++ /dev/null
@@ -1,143 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import static com.nearinfinity.blur.utils.BlurConstants.RECORD_ID;
-import static com.nearinfinity.blur.utils.BlurConstants.ROW_ID;
-import static com.nearinfinity.blur.utils.BlurConstants.SEP;
-import static com.nearinfinity.blur.utils.BlurConstants.SUPER;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-
-import com.nearinfinity.blur.analysis.BlurAnalyzer;
-import com.nearinfinity.blur.analysis.FieldConverterUtil;
-import com.nearinfinity.blur.thrift.generated.Column;
-import com.nearinfinity.blur.thrift.generated.Record;
-import com.nearinfinity.blur.thrift.generated.Row;
-
-public class RowIndexWriter {
-
-  private BlurAnalyzer _analyzer;
-  private IndexWriter _indexWriter;
-  private boolean primeDocSet;
-  private StringBuilder builder = new StringBuilder();
-
-  public RowIndexWriter(IndexWriter indexWriter, BlurAnalyzer analyzer) {
-    _indexWriter = indexWriter;
-    _analyzer = analyzer;
-  }
-
-  public void add(boolean wal, Row row) throws IOException {
-    if (row == null || row.id == null) {
-      throw new NullPointerException();
-    }
-    append(wal, row, false);
-  }
-
-  public void replace(boolean wal, Row row) throws IOException {
-    if (row == null || row.id == null) {
-      throw new NullPointerException();
-    }
-    append(wal, row, true);
-  }
-
-  private void append(boolean wal, Row row, boolean replace) throws IOException {
-    primeDocSet = false;
-    List<Document> documents = new ArrayList<Document>();
-    for (Record record : row.records) {
-      convert(row.id, record, documents);
-    }
-    if (replace) {
-      _indexWriter.updateDocuments(new Term(ROW_ID, row.id), documents, _analyzer);
-    } else {
-      _indexWriter.addDocuments(documents, _analyzer);
-    }
-  }
-
-  private void convert(String rowId, Record record, List<Document> documents) throws IOException {
-    if (record == null) {
-      return;
-    }
-    String recordId = record.recordId;
-    if (recordId == null) {
-      throw new NullPointerException("Record id is null.");
-    }
-    String family = record.getFamily();
-    if (family == null) {
-      throw new NullPointerException("Family is null.");
-    }
-    Document document = new Document();
-    document.add(new Field(ROW_ID, rowId, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
-    document.add(new Field(RECORD_ID, recordId, Store.YES, Index.NOT_ANALYZED_NO_NORMS));
-    if (addColumns(document, _analyzer, builder, family, record.columns)) {
-      FieldConverterUtil.convert(document, _analyzer);
-      if (!primeDocSet) {
-        document.add(BlurConstants.PRIME_DOC_FIELD);
-        primeDocSet = true;
-      }
-      documents.add(document);
-    }
-  }
-
-  public static boolean addColumns(Document document, BlurAnalyzer analyzer, StringBuilder builder, String columnFamily, Iterable<Column> set) {
-    if (set == null) {
-      return false;
-    }
-    builder.setLength(0);
-    OUTER: for (Column column : set) {
-      String name = column.getName();
-      String value = column.value;
-      if (value == null || name == null) {
-        continue OUTER;
-      }
-      String fieldName = getFieldName(columnFamily, name);
-      Store store = analyzer.getStore(fieldName);
-      Index index = analyzer.getIndex(fieldName);
-      boolean fullText = analyzer.isFullTextField(fieldName);
-      Set<String> subFieldNames = analyzer.getSubIndexNames(fieldName);
-      document.add(new Field(fieldName, value, store, index));
-      if (fullText) {
-        builder.append(value).append(' ');
-      }
-      if (subFieldNames != null) {
-        for (String subFieldName : subFieldNames) {
-          document.add(new Field(subFieldName, value, Store.NO, index));
-        }
-      }
-    }
-    if (builder.length() != 0) {
-      String superValue = builder.toString();
-      document.add(new Field(SUPER, superValue, Store.NO, Index.ANALYZED_NO_NORMS));
-    }
-    return true;
-  }
-
-  public static String getFieldName(String columnFamily, String name) {
-    return columnFamily + SEP + name;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/com/nearinfinity/blur/utils/TermDocIterable.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/TermDocIterable.java b/src/blur-core/src/main/java/com/nearinfinity/blur/utils/TermDocIterable.java
deleted file mode 100644
index ab28bc1..0000000
--- a/src/blur-core/src/main/java/com/nearinfinity/blur/utils/TermDocIterable.java
+++ /dev/null
@@ -1,91 +0,0 @@
-package com.nearinfinity.blur.utils;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermDocs;
-
-public class TermDocIterable implements Iterable<Document> {
-
-  private TermDocs termDocs;
-  private IndexReader reader;
-  private FieldSelector fieldSelector;
-
-  public TermDocIterable(TermDocs termDocs, IndexReader reader, FieldSelector fieldSelector) {
-    this.termDocs = termDocs;
-    this.reader = reader;
-    this.fieldSelector = fieldSelector;
-  }
-
-  @Override
-  public Iterator<Document> iterator() {
-    return new Iterator<Document>() {
-      private boolean hasNext = getNext();
-
-      @Override
-      public boolean hasNext() {
-        return hasNext;
-      }
-
-      @Override
-      public Document next() {
-        Document doc;
-        try {
-          doc = getDoc();
-          hasNext = getNext();
-        } catch (IOException e) {
-          throw new RuntimeException(e);
-        }
-        return doc;
-      }
-
-      @Override
-      public void remove() {
-
-      }
-    };
-  }
-
-  private Document getDoc() throws IOException {
-    return reader.document(termDocs.doc(), fieldSelector);
-  }
-
-  private boolean getNext() {
-    try {
-      boolean next = termDocs.next();
-      if (!next) {
-        termDocs.close();
-        return next;
-      }
-      while (reader.isDeleted(termDocs.doc())) {
-        next = termDocs.next();
-      }
-      if (!next) {
-        termDocs.close();
-      }
-      return next;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
new file mode 100644
index 0000000..6d19c69
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/BlurAnalyzer.java
@@ -0,0 +1,309 @@
+package org.apache.blur.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.apache.blur.lucene.LuceneConstant.LUCENE_VERSION;
+import static org.apache.blur.utils.BlurConstants.PRIME_DOC;
+import static org.apache.blur.utils.BlurConstants.RECORD_ID;
+import static org.apache.blur.utils.BlurConstants.ROW_ID;
+import static org.apache.blur.utils.BlurConstants.SUPER;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.lang.reflect.Constructor;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.blur.thrift.generated.AlternateColumnDefinition;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.ColumnDefinition;
+import org.apache.blur.thrift.generated.ColumnFamilyDefinition;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.KeywordAnalyzer;
+import org.apache.lucene.analysis.PerFieldAnalyzerWrapper;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.util.Version;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.apache.thrift.transport.TMemoryBuffer;
+import org.apache.thrift.transport.TMemoryInputTransport;
+
+
+public class BlurAnalyzer extends Analyzer {
+
+  private static final String STANDARD = "org.apache.lucene.analysis.standard.StandardAnalyzer";
+
+  public static final BlurAnalyzer BLANK_ANALYZER = new BlurAnalyzer(new KeywordAnalyzer());
+
+  private static Map<String, Class<? extends Analyzer>> aliases = new HashMap<String, Class<? extends Analyzer>>();
+
+  private Map<String, Store> _storeMap = new HashMap<String, Store>();
+  private Map<String, Set<String>> _subIndexNameLookups = new HashMap<String, Set<String>>();
+  private Set<String> _fullTextFields = new HashSet<String>();
+  private AnalyzerDefinition _analyzerDefinition;
+  private PerFieldAnalyzerWrapper _wrapper;
+  private Analyzer _fullTextAnalyzer = new StandardAnalyzer(LUCENE_VERSION);
+
+  private HashMap<String, Analyzer> _analyzers;
+
+  public void addSubField(String name) {
+    int lastIndexOf = name.lastIndexOf('.');
+    String mainFieldName = name.substring(0, lastIndexOf);
+    Set<String> set = _subIndexNameLookups.get(mainFieldName);
+    if (set == null) {
+      set = new TreeSet<String>();
+      _subIndexNameLookups.put(mainFieldName, set);
+    }
+    set.add(name);
+  }
+
+  public Set<String> getSubIndexNames(String indexName) {
+    return _subIndexNameLookups.get(indexName);
+  }
+
+  public BlurAnalyzer(Analyzer analyzer) {
+    _analyzerDefinition = new AnalyzerDefinition();
+    _wrapper = new PerFieldAnalyzerWrapper(analyzer);
+  }
+
+  public BlurAnalyzer(AnalyzerDefinition analyzerDefinition) {
+    _analyzerDefinition = analyzerDefinition;
+    ColumnDefinition defaultDefinition = analyzerDefinition.getDefaultDefinition();
+    String fullTextAnalyzerClassName = analyzerDefinition.fullTextAnalyzerClassName;
+    if (fullTextAnalyzerClassName != null) {
+      _fullTextAnalyzer = getAnalyzerByClassName(fullTextAnalyzerClassName, aliases);
+    }
+    if (defaultDefinition == null) {
+      defaultDefinition = new ColumnDefinition(STANDARD, true, null);
+      analyzerDefinition.setDefaultDefinition(defaultDefinition);
+    }
+    Analyzer defaultAnalyzer = getAnalyzerByClassName(defaultDefinition.getAnalyzerClassName(), aliases);
+    KeywordAnalyzer keywordAnalyzer = new KeywordAnalyzer();
+    _analyzers = new HashMap<String, Analyzer>();
+    _analyzers.put(ROW_ID, keywordAnalyzer);
+    _analyzers.put(RECORD_ID, keywordAnalyzer);
+    _analyzers.put(PRIME_DOC, keywordAnalyzer);
+    _analyzers.put(SUPER, _fullTextAnalyzer);
+    load(_analyzers);
+    _wrapper = new PerFieldAnalyzerWrapper(defaultAnalyzer, _analyzers);
+  }
+
+  public FieldConverter getFieldConverter(String name) {
+    if (_analyzers == null) {
+      return null;
+    }
+    Analyzer analyzer = _analyzers.get(name);
+    if (analyzer != null && analyzer instanceof FieldConverter) {
+      return (FieldConverter) analyzer;
+    }
+    return null;
+  }
+
+  private void load(Map<String, Analyzer> analyzers) {
+    Map<String, ColumnFamilyDefinition> familyDefinitions = _analyzerDefinition.columnFamilyDefinitions;
+    if (familyDefinitions != null) {
+      for (String family : familyDefinitions.keySet()) {
+        ColumnFamilyDefinition familyDefinition = familyDefinitions.get(family);
+        load(family, familyDefinition, analyzers);
+      }
+    }
+  }
+
+  private void load(String family, ColumnFamilyDefinition familyDefinition, Map<String, Analyzer> analyzers) {
+    Map<String, ColumnDefinition> columnDefinitions = familyDefinition.columnDefinitions;
+    for (String column : columnDefinitions.keySet()) {
+      ColumnDefinition columnDefinition = columnDefinitions.get(column);
+      load(family, familyDefinition, column, columnDefinition, analyzers);
+    }
+  }
+
+  private void load(String family, ColumnFamilyDefinition familyDefinition, String column, ColumnDefinition columnDefinition, Map<String, Analyzer> analyzers) {
+    Map<String, AlternateColumnDefinition> alternateColumnDefinitions = columnDefinition.alternateColumnDefinitions;
+    if (alternateColumnDefinitions != null) {
+      for (String subColumn : alternateColumnDefinitions.keySet()) {
+        AlternateColumnDefinition alternateColumnDefinition = alternateColumnDefinitions.get(subColumn);
+        load(family, familyDefinition, column, columnDefinition, subColumn, alternateColumnDefinition, analyzers);
+      }
+    }
+    String fieldName = family + "." + column;
+    Analyzer analyzer = getAnalyzerByClassName(columnDefinition.getAnalyzerClassName(), aliases);
+    analyzers.put(fieldName, analyzer);
+    if (columnDefinition.isFullTextIndex()) {
+      _fullTextFields.add(fieldName);
+    }
+  }
+
+  private void load(String family, ColumnFamilyDefinition familyDefinition, String column, ColumnDefinition columnDefinition, String subColumn,
+      AlternateColumnDefinition alternateColumnDefinition, Map<String, Analyzer> analyzers) {
+    String fieldName = family + "." + column + "." + subColumn;
+    Analyzer analyzer = getAnalyzerByClassName(alternateColumnDefinition.getAnalyzerClassName(), aliases);
+    analyzers.put(fieldName, analyzer);
+    putStore(fieldName, Store.NO);
+    addSubField(fieldName);
+  }
+
+  public void putStore(String name, Store store) {
+    _storeMap.put(name, store);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Analyzer getAnalyzerByClassName(String className, Map<String, Class<? extends Analyzer>> aliases) {
+    if (FieldConverterUtil.isType(className)) {
+      return FieldConverterUtil.getAnalyzer(className);
+    }
+    try {
+      Class<? extends Analyzer> clazz = aliases.get(className);
+      if (clazz == null) {
+        clazz = (Class<? extends Analyzer>) Class.forName(className);
+      }
+      try {
+        return (Analyzer) clazz.newInstance();
+      } catch (Exception e) {
+        Constructor<?> constructor = clazz.getConstructor(new Class[] { Version.class });
+        return (Analyzer) constructor.newInstance(LUCENE_VERSION);
+      }
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public Store getStore(String indexName) {
+    Store store = _storeMap.get(indexName);
+    if (store == null) {
+      return Store.YES;
+    }
+    return store;
+  }
+
+  public Index getIndex(String indexName) {
+    return Index.ANALYZED_NO_NORMS;
+  }
+
+  public String toJSON() {
+    TMemoryBuffer trans = new TMemoryBuffer(1024);
+    TJSONProtocol protocol = new TJSONProtocol(trans);
+    try {
+      _analyzerDefinition.write(protocol);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+    trans.close();
+    byte[] array = trans.getArray();
+    return new String(array, 0, trans.length());
+  }
+
+  public boolean isFullTextField(String fieldName) {
+    ColumnDefinition defaultDefinition = _analyzerDefinition.getDefaultDefinition();
+    if (defaultDefinition != null && defaultDefinition.fullTextIndex) {
+      return true;
+    }
+    return _fullTextFields.contains(fieldName);
+  }
+
+  public AnalyzerDefinition getAnalyzerDefinition() {
+    return _analyzerDefinition;
+  }
+
+  public void close() {
+    _wrapper.close();
+  }
+
+  public int getOffsetGap(Fieldable field) {
+    return _wrapper.getOffsetGap(field);
+  }
+
+  public int getPositionIncrementGap(String fieldName) {
+    return _wrapper.getPositionIncrementGap(fieldName);
+  }
+
+  public final TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
+    return _wrapper.reusableTokenStream(fieldName, reader);
+  }
+
+  public final TokenStream tokenStream(String fieldName, Reader reader) {
+    return _wrapper.tokenStream(fieldName, reader);
+  }
+
+  public static BlurAnalyzer create(File file) throws IOException {
+    FileInputStream inputStream = new FileInputStream(file);
+    try {
+      return create(inputStream);
+    } finally {
+      inputStream.close();
+    }
+  }
+
+  public static BlurAnalyzer create(InputStream inputStream) throws IOException {
+    TMemoryInputTransport trans = new TMemoryInputTransport(getBytes(inputStream));
+    TJSONProtocol protocol = new TJSONProtocol(trans);
+    AnalyzerDefinition analyzerDefinition = new AnalyzerDefinition();
+    try {
+      analyzerDefinition.read(protocol);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+    trans.close();
+    return new BlurAnalyzer(analyzerDefinition);
+  }
+
+  private static byte[] getBytes(InputStream inputStream) throws IOException {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    byte[] buffer = new byte[1024];
+    int num;
+    while ((num = inputStream.read(buffer)) != -1) {
+      outputStream.write(buffer, 0, num);
+    }
+    inputStream.close();
+    outputStream.close();
+    return outputStream.toByteArray();
+  }
+
+  public static BlurAnalyzer create(String jsonStr) throws IOException {
+    InputStream inputStream = new ByteArrayInputStream(jsonStr.getBytes());
+    try {
+      return create(inputStream);
+    } finally {
+      inputStream.close();
+    }
+  }
+
+  public static BlurAnalyzer create(Path path) throws IOException {
+    FileSystem fileSystem = FileSystem.get(path.toUri(), new Configuration());
+    FSDataInputStream inputStream = fileSystem.open(path);
+    try {
+      return create(inputStream);
+    } finally {
+      inputStream.close();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverter.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverter.java b/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverter.java
new file mode 100644
index 0000000..8477cc7
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverter.java
@@ -0,0 +1,24 @@
+package org.apache.blur.analysis;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.lucene.document.Fieldable;
+
+public interface FieldConverter {
+
+  Fieldable convert(Fieldable orig);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverterUtil.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverterUtil.java b/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverterUtil.java
new file mode 100644
index 0000000..e0c4642
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/FieldConverterUtil.java
@@ -0,0 +1,81 @@
+package org.apache.blur.analysis;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.List;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Fieldable;
+
+public class FieldConverterUtil {
+
+  private static final String LONG = "long";
+
+  /**
+   * This method runs the converter on each of the fields in the document and
+   * returns the a new document.
+   * 
+   * @param document
+   *          the original document.
+   * @param converter
+   *          the converter.
+   * @return the original document.
+   */
+  public static Document convert(Document document, FieldConverter converter) {
+    List<Fieldable> fields = document.getFields();
+    int size = fields.size();
+    for (int i = 0; i < size; i++) {
+      Fieldable origField = fields.get(i);
+      Fieldable newField = converter.convert(origField);
+      if (newField != null) {
+        fields.set(i, newField);
+      }
+    }
+    return document;
+  }
+
+  public static Document convert(Document document, BlurAnalyzer analyzer) {
+    List<Fieldable> fields = document.getFields();
+    int size = fields.size();
+    for (int i = 0; i < size; i++) {
+      Fieldable origField = fields.get(i);
+      FieldConverter converter = analyzer.getFieldConverter(origField.name());
+      if (converter != null) {
+        Fieldable newField = converter.convert(origField);
+        if (newField != null) {
+          fields.set(i, newField);
+        }
+      }
+    }
+    return document;
+  }
+
+  public static boolean isType(String type) {
+    if (type.startsWith(LONG)) {
+      return true;
+    }
+    return false;
+  }
+
+  public static Analyzer getAnalyzer(String type) {
+    if (type.startsWith(LONG)) {
+      return new LongAnalyzer(type);
+    }
+    throw new RuntimeException("Type [" + type + "] not found.");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/33df9310/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java b/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
new file mode 100644
index 0000000..a7ea6f3
--- /dev/null
+++ b/src/blur-core/src/main/java/org/apache/blur/analysis/LongAnalyzer.java
@@ -0,0 +1,62 @@
+package org.apache.blur.analysis;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.Reader;
+
+import org.apache.lucene.analysis.KeywordTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.util.NumericUtils;
+
+public class LongAnalyzer extends ReusableAnalyzerBase implements FieldConverter {
+
+  private static final String TYPE = "long";
+
+  private int precisionStepDefault = NumericUtils.PRECISION_STEP_DEFAULT;
+
+  public LongAnalyzer(String typeStr) {
+    if (typeStr.startsWith(TYPE)) {
+      int index = typeStr.indexOf(',');
+      if (index > 0) {
+        String s = typeStr.substring(index + 1);
+        try {
+          precisionStepDefault = Integer.parseInt(s);
+        } catch (NumberFormatException e) {
+          throw new RuntimeException("Can not parser [" + s + "] into an integer for the precisionStepDefault.");
+        }
+      }
+    } else {
+      throw new RuntimeException("Long type can not parser [" + typeStr + "]");
+    }
+  }
+
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    return new TokenStreamComponents(new KeywordTokenizer(reader));
+  }
+
+  @Override
+  public Fieldable convert(Fieldable fieldable) {
+    long value = Long.parseLong(fieldable.stringValue().trim());
+    NumericField field = new NumericField(fieldable.name(), precisionStepDefault, fieldable.isStored() ? Store.YES : Store.NO, true);
+    field.setLongValue(value);
+    return field;
+  }
+
+}


Mime
View raw message