incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [44/51] [partial] Fixed BLUR-126.
Date Thu, 06 Jun 2013 18:58:16 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
new file mode 100644
index 0000000..029ee6e
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/BlurUtil.java
@@ -0,0 +1,665 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.apache.blur.metrics.MetricsConstants.BLUR;
+import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR;
+import static org.apache.blur.metrics.MetricsConstants.THRIFT_CALLS;
+import static org.apache.blur.utils.BlurConstants.SHARD_PREFIX;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLongArray;
+import java.util.regex.Pattern;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.manager.clusterstatus.ZookeeperPathConstants;
+import org.apache.blur.manager.results.BlurResultIterable;
+import org.apache.blur.thirdparty.thrift_0_9_0.TBase;
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TMemoryBuffer;
+import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResult;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.FetchResult;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.Row;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+import org.apache.blur.thrift.generated.Selector;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.Code;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Histogram;
+import com.yammer.metrics.core.MetricName;
+
+public class BlurUtil {
+
+  private static final Object[] EMPTY_OBJECT_ARRAY = new Object[] {};
+  private static final Class<?>[] EMPTY_PARAMETER_TYPES = new Class[] {};
+  private static final Log LOG = LogFactory.getLog(BlurUtil.class);
+  private static final String UNKNOWN = "UNKNOWN";
+  private static Pattern validator = Pattern.compile("^[a-zA-Z0-9\\_\\-]+$");
+
+  @SuppressWarnings("unchecked")
+  public static <T extends Iface> T recordMethodCallsAndAverageTimes(final T t, Class<T> clazz) {
+    final Map<String, Histogram> histogramMap = new ConcurrentHashMap<String, Histogram>();
+    Method[] declaredMethods = Iface.class.getDeclaredMethods();
+    for (Method m : declaredMethods) {
+      String name = m.getName();
+      histogramMap.put(name, Metrics.newHistogram(new MetricName(ORG_APACHE_BLUR, BLUR, name, THRIFT_CALLS)));
+    }
+    InvocationHandler handler = new InvocationHandler() {
+      @Override
+      public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+        long start = System.nanoTime();
+        try {
+          return method.invoke(t, args);
+        } catch (InvocationTargetException e) {
+          throw e.getTargetException();
+        } finally {
+          long end = System.nanoTime();
+          String name = method.getName();
+          Histogram histogram = histogramMap.get(name);
+          histogram.update((end - start) / 1000);
+        }
+      }
+    };
+    return (T) Proxy.newProxyInstance(clazz.getClassLoader(), new Class[] { clazz }, handler);
+  }
+
+  public static void setupZookeeper(ZooKeeper zookeeper) throws KeeperException, InterruptedException {
+    setupZookeeper(zookeeper, null);
+  }
+
+  public synchronized static void setupZookeeper(ZooKeeper zookeeper, String cluster) throws KeeperException,
+      InterruptedException {
+    BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getBasePath());
+    BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getOnlineControllersPath());
+    BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getClustersPath());
+    if (cluster != null) {
+      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getClusterPath(cluster));
+      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getSafemodePath(cluster));
+      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getRegisteredShardsPath(cluster));
+      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getOnlineShardsPath(cluster));
+      BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getTablesPath(cluster));
+    }
+  }
+
+  public static void createIfMissing(ZooKeeper zookeeper, String path) throws KeeperException, InterruptedException {
+    if (zookeeper.exists(path, false) == null) {
+      try {
+        zookeeper.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+      } catch (KeeperException e) {
+        if (e.code() == Code.NODEEXISTS) {
+          return;
+        }
+        throw e;
+      }
+    }
+  }
+
+  public static List<Long> getList(AtomicLongArray atomicLongArray) {
+    if (atomicLongArray == null) {
+      return null;
+    }
+    List<Long> counts = new ArrayList<Long>(atomicLongArray.length());
+    for (int i = 0; i < atomicLongArray.length(); i++) {
+      counts.add(atomicLongArray.get(i));
+    }
+    return counts;
+  }
+
+  public static void quietClose(Object... close) {
+    if (close == null) {
+      return;
+    }
+    for (Object object : close) {
+      if (object != null) {
+        close(object);
+      }
+    }
+  }
+
+  private static void close(Object object) {
+    Class<? extends Object> clazz = object.getClass();
+    Method method;
+    try {
+      method = clazz.getMethod("close", EMPTY_PARAMETER_TYPES);
+    } catch (SecurityException e) {
+      throw new RuntimeException(e);
+    } catch (NoSuchMethodException e) {
+      return;
+    }
+    try {
+      method.invoke(object, EMPTY_OBJECT_ARRAY);
+    } catch (Exception e) {
+      LOG.error("Error while trying to close object [{0}]", e, object);
+    }
+  }
+
+  public static byte[] toBytes(Serializable serializable) {
+    try {
+      ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+      ObjectOutputStream stream = new ObjectOutputStream(outputStream);
+      stream.writeObject(serializable);
+      stream.close();
+      return outputStream.toByteArray();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static Serializable fromBytes(byte[] bs) {
+    ObjectInputStream stream = null;
+    try {
+      stream = new ObjectInputStream(new ByteArrayInputStream(bs));
+      return (Serializable) stream.readObject();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    } finally {
+      if (stream != null) {
+        try {
+          stream.close();
+        } catch (IOException e) {
+          // eat
+        }
+      }
+    }
+  }
+
+  public static List<Long> toList(AtomicLongArray atomicLongArray) {
+    if (atomicLongArray == null) {
+      return null;
+    }
+    int length = atomicLongArray.length();
+    List<Long> result = new ArrayList<Long>(length);
+    for (int i = 0; i < length; i++) {
+      result.add(atomicLongArray.get(i));
+    }
+    return result;
+  }
+
+  public static AtomicLongArray getAtomicLongArraySameLengthAsList(List<?> list) {
+    if (list == null) {
+      return null;
+    }
+    return new AtomicLongArray(list.size());
+  }
+
+  public static BlurResults convertToHits(BlurResultIterable hitsIterable, BlurQuery query,
+      AtomicLongArray facetCounts, ExecutorService executor, Selector selector, final Iface iface, final String table)
+      throws InterruptedException, ExecutionException {
+    BlurResults results = new BlurResults();
+    results.setTotalResults(hitsIterable.getTotalResults());
+    results.setShardInfo(hitsIterable.getShardInfo());
+    if (query.minimumNumberOfResults > 0) {
+      hitsIterable.skipTo(query.start);
+      int count = 0;
+      Iterator<BlurResult> iterator = hitsIterable.iterator();
+      while (iterator.hasNext() && count < query.fetch) {
+        results.addToResults(iterator.next());
+        count++;
+      }
+    }
+    if (results.results == null) {
+      results.results = new ArrayList<BlurResult>();
+    }
+    if (facetCounts != null) {
+      results.facetCounts = BlurUtil.toList(facetCounts);
+    }
+    if (selector != null) {
+      List<Future<FetchResult>> futures = new ArrayList<Future<FetchResult>>();
+      for (int i = 0; i < results.results.size(); i++) {
+        BlurResult result = results.results.get(i);
+        final Selector s = new Selector(selector);
+        s.setLocationId(result.locationId);
+        futures.add(executor.submit(new Callable<FetchResult>() {
+          @Override
+          public FetchResult call() throws Exception {
+            return iface.fetchRow(table, s);
+          }
+        }));
+      }
+      for (int i = 0; i < results.results.size(); i++) {
+        Future<FetchResult> future = futures.get(i);
+        BlurResult result = results.results.get(i);
+        result.setFetchResult(future.get());
+      }
+    }
+    results.query = query;
+    results.query.selector = selector;
+    return results;
+  }
+
+  public static void setStartTime(BlurQuery query) {
+    if (query.startTime == 0) {
+      query.startTime = System.currentTimeMillis();
+    }
+  }
+
+  public static String getVersion() {
+    String path = "/META-INF/maven/org.apache.blur/blur-core/pom.properties";
+    InputStream inputStream = BlurUtil.class.getResourceAsStream(path);
+    if (inputStream == null) {
+      return UNKNOWN;
+    }
+    Properties prop = new Properties();
+    try {
+      prop.load(inputStream);
+    } catch (IOException e) {
+      LOG.error("Unknown error while getting version.", e);
+      return UNKNOWN;
+    }
+    Object verison = prop.get("version");
+    if (verison == null) {
+      return UNKNOWN;
+    }
+    return verison.toString();
+  }
+
+  public static void unlockForSafeMode(ZooKeeper zookeeper, String lockPath) throws InterruptedException,
+      KeeperException {
+    zookeeper.delete(lockPath, -1);
+    LOG.info("Lock released.");
+  }
+
+  public static String lockForSafeMode(ZooKeeper zookeeper, String nodeName, String cluster) throws KeeperException,
+      InterruptedException {
+    LOG.info("Getting safe mode lock.");
+    final Object lock = new Object();
+    String blurSafemodePath = ZookeeperPathConstants.getSafemodePath(cluster);
+    String newPath = zookeeper.create(blurSafemodePath + "/safemode-", nodeName.getBytes(), Ids.OPEN_ACL_UNSAFE,
+        CreateMode.EPHEMERAL_SEQUENTIAL);
+    Watcher watcher = new Watcher() {
+      @Override
+      public void process(WatchedEvent event) {
+        synchronized (lock) {
+          lock.notifyAll();
+        }
+      }
+    };
+    while (true) {
+      synchronized (lock) {
+        List<String> children = new ArrayList<String>(zookeeper.getChildren(blurSafemodePath, watcher));
+        Collections.sort(children);
+        if (newPath.equals(blurSafemodePath + "/" + children.get(0))) {
+          LOG.info("Lock aquired.");
+          return newPath;
+        } else {
+          lock.wait(BlurConstants.ZK_WAIT_TIME);
+        }
+      }
+    }
+  }
+
+  public static String getShardName(int id) {
+    return getShardName(BlurConstants.SHARD_PREFIX, id);
+  }
+
+  public static String getShardName(String prefix, int id) {
+    return prefix + buffer(id, 8);
+  }
+
+  private static String buffer(int value, int length) {
+    String str = Integer.toString(value);
+    while (str.length() < length) {
+      str = "0" + str;
+    }
+    return str;
+  }
+
+  public static String humanizeTime(long time, TimeUnit unit) {
+    long seconds = unit.toSeconds(time);
+    long hours = getHours(seconds);
+    seconds = seconds - TimeUnit.HOURS.toSeconds(hours);
+    long minutes = getMinutes(seconds);
+    seconds = seconds - TimeUnit.MINUTES.toSeconds(minutes);
+    return humanizeTime(hours, minutes, seconds);
+  }
+
+  public static String humanizeTime(long hours, long minutes, long seconds) {
+    StringBuilder builder = new StringBuilder();
+    if (hours == 0 && minutes != 0) {
+      addMinutes(builder, minutes);
+    } else if (hours != 0) {
+      addHours(builder, hours);
+      addMinutes(builder, minutes);
+    }
+    addSeconds(builder, seconds);
+    return builder.toString().trim();
+  }
+
+  private static void addHours(StringBuilder builder, long hours) {
+    builder.append(hours).append(" hours ");
+  }
+
+  private static void addMinutes(StringBuilder builder, long minutes) {
+    builder.append(minutes).append(" minutes ");
+  }
+
+  private static void addSeconds(StringBuilder builder, long seconds) {
+    builder.append(seconds).append(" seconds ");
+  }
+
+  private static long getMinutes(long seconds) {
+    return seconds / TimeUnit.MINUTES.toSeconds(1);
+  }
+
+  private static long getHours(long seconds) {
+    return seconds / TimeUnit.HOURS.toSeconds(1);
+  }
+
+  public static long getMemoryUsage(IndexReader r) {
+    long sizeOf = RamUsageEstimator.sizeOf(r);
+    return sizeOf;
+  }
+
+  public static void createPath(ZooKeeper zookeeper, String path, byte[] data) throws KeeperException,
+      InterruptedException {
+    zookeeper.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+  }
+
+  public static void setupFileSystem(String uri, int shardCount) throws IOException {
+    Path tablePath = new Path(uri);
+    FileSystem fileSystem = FileSystem.get(tablePath.toUri(), new Configuration());
+    if (createPath(fileSystem, tablePath)) {
+      LOG.info("Table uri existed.");
+      validateShardCount(shardCount, fileSystem, tablePath);
+    }
+    for (int i = 0; i < shardCount; i++) {
+      String shardName = BlurUtil.getShardName(SHARD_PREFIX, i);
+      Path shardPath = new Path(tablePath, shardName);
+      createPath(fileSystem, shardPath);
+    }
+  }
+
+  public static void validateShardCount(int shardCount, FileSystem fileSystem, Path tablePath) throws IOException {
+    // Check that all the directories that should be are in fact there.
+    for (int i = 0; i < shardCount; i++) {
+      Path path = new Path(tablePath, BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i));
+      if (!fileSystem.exists(path)) {
+        LOG.error("Path [{0}] for shard [{1}] does not exist.", path, i);
+        throw new RuntimeException("Path [" + path + "] for shard [" + i + "] does not exist.");
+      }
+    }
+
+    FileStatus[] listStatus = fileSystem.listStatus(tablePath);
+    for (FileStatus fs : listStatus) {
+      Path path = fs.getPath();
+      String name = path.getName();
+      if (name.startsWith(SHARD_PREFIX)) {
+        int index = name.indexOf('-');
+        String shardIndexStr = name.substring(index + 1);
+        int shardIndex = Integer.parseInt(shardIndexStr);
+        if (shardIndex >= shardCount) {
+          LOG.error("Number of directories in table path [" + path + "] exceeds definition of [" + shardCount
+              + "] shard count.");
+          throw new RuntimeException("Number of directories in table path [" + path + "] exceeds definition of ["
+              + shardCount + "] shard count.");
+        }
+      }
+    }
+  }
+
+  public static boolean createPath(FileSystem fileSystem, Path path) throws IOException {
+    if (!fileSystem.exists(path)) {
+      LOG.info("Path [{0}] does not exist, creating.", path);
+      fileSystem.mkdirs(path);
+      return false;
+    }
+    return true;
+  }
+
+  public static int zeroCheck(int i, String message) {
+    if (i < 1) {
+      throw new RuntimeException(message);
+    }
+    return i;
+  }
+
+  public static <T> T nullCheck(T t, String message) {
+    if (t == null) {
+      throw new NullPointerException(message);
+    }
+    return t;
+  }
+
+  @SuppressWarnings("unchecked")
+  public static <T> T getInstance(String className, Class<T> c) {
+    Class<?> clazz;
+    try {
+      clazz = Class.forName(className);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+    try {
+      return (T) configure(clazz.newInstance());
+    } catch (InstantiationException e) {
+      throw new RuntimeException(e);
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static <T> T configure(T t) {
+    if (t instanceof Configurable) {
+      Configurable configurable = (Configurable) t;
+      configurable.setConf(new Configuration());
+    }
+    return t;
+  }
+
+  public static byte[] read(TBase<?, ?> base) {
+    if (base == null) {
+      return null;
+    }
+    TMemoryBuffer trans = new TMemoryBuffer(1024);
+    TJSONProtocol protocol = new TJSONProtocol(trans);
+    try {
+      base.write(protocol);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+    trans.close();
+    byte[] buf = new byte[trans.length()];
+    System.arraycopy(trans.getArray(), 0, buf, 0, trans.length());
+    return buf;
+  }
+
+  public static void write(byte[] data, TBase<?, ?> base) {
+    nullCheck(null, "Data cannot be null.");
+    TMemoryBuffer trans = new TMemoryBuffer(1024);
+    TJSONProtocol protocol = new TJSONProtocol(trans);
+    try {
+      trans.write(data);
+      base.read(protocol);
+    } catch (TException e) {
+      throw new RuntimeException(e);
+    }
+    trans.close();
+  }
+
+  public static void removeAll(ZooKeeper zooKeeper, String path) throws KeeperException, InterruptedException {
+    List<String> list = zooKeeper.getChildren(path, false);
+    for (String p : list) {
+      removeAll(zooKeeper, path + "/" + p);
+    }
+    LOG.info("Removing path [{0}]", path);
+    zooKeeper.delete(path, -1);
+  }
+
+  public static void removeIndexFiles(String uri) throws IOException {
+    Path tablePath = new Path(uri);
+    FileSystem fileSystem = FileSystem.get(tablePath.toUri(), new Configuration());
+    fileSystem.delete(tablePath, true);
+  }
+
+  public static RowMutation toRowMutation(String table, Row row) {
+    RowMutation rowMutation = new RowMutation();
+    rowMutation.setRowId(row.getId());
+    rowMutation.setTable(table);
+    rowMutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+    List<Record> records = row.getRecords();
+    for (Record record : records) {
+      rowMutation.addToRecordMutations(toRecordMutation(record));
+    }
+    return rowMutation;
+  }
+
+  public static RecordMutation toRecordMutation(Record record) {
+    RecordMutation recordMutation = new RecordMutation();
+    recordMutation.setRecord(record);
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+    return recordMutation;
+  }
+  
+  public static int countDocuments(IndexReader reader, Term term) throws IOException {
+    TermQuery query = new TermQuery(term);
+    IndexSearcher indexSearcher = new IndexSearcher(reader);
+    TopDocs topDocs = indexSearcher.search(query, 1);
+    return topDocs.totalHits;
+  }
+
+  /**
+   * NOTE: This is a potentially dangerous call, it will return all the
+   * documents that match the term.
+   * 
+   * @param selector
+   * 
+   * @throws IOException
+   */
+  public static List<Document> fetchDocuments(IndexReader reader, Term term,
+      ResetableDocumentStoredFieldVisitor fieldSelector, Selector selector) throws IOException {
+    IndexSearcher indexSearcher = new IndexSearcher(reader);
+		int docFreq = reader.docFreq(term);
+		BooleanQuery booleanQueryForFamily = null;
+		BooleanQuery booleanQuery = null;
+		if (selector.getColumnFamiliesToFetchSize() > 0) {
+			booleanQueryForFamily = new BooleanQuery();
+			for (String familyName : selector.getColumnFamiliesToFetch()) {
+				booleanQueryForFamily.add(new TermQuery(new Term(
+						BlurConstants.FAMILY, familyName)),
+						BooleanClause.Occur.SHOULD);
+			}
+			booleanQuery = new BooleanQuery();
+			booleanQuery.add(new TermQuery(term), BooleanClause.Occur.MUST);
+			booleanQuery.add(booleanQueryForFamily, BooleanClause.Occur.MUST);
+		}
+		Query query = booleanQuery == null ? new TermQuery(term) : booleanQuery;
+		TopDocs topDocs = indexSearcher.search(query, docFreq);
+		int totalHits = topDocs.totalHits;
+		List<Document> docs = new ArrayList<Document>();
+
+    int start = selector.getStartRecord();
+    int end = selector.getMaxRecordsToFetch() + start;
+
+    for (int i = start; i < end; i++) {
+      if (i >= totalHits) {
+        break;
+      }
+      int doc = topDocs.scoreDocs[i].doc;
+      indexSearcher.doc(doc, fieldSelector);
+      docs.add(fieldSelector.getDocument());
+      fieldSelector.reset();
+    }
+    return docs;
+  }
+
+  public static AtomicReader getAtomicReader(IndexReader reader) throws IOException {
+    return SlowCompositeReaderWrapper.wrap(reader);
+  }
+
+  public static int getShardIndex(String shard) {
+    int index = shard.indexOf('-');
+    return Integer.parseInt(shard.substring(index + 1));
+  }
+  
+	public static void validateRowIdAndRecord(String rowId, Record record) {
+		if (!validator.matcher(record.family).matches()) {
+			throw new IllegalArgumentException("Invalid column family name [ " + record.family + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+		}
+
+		for (Column column : record.getColumns()) {
+			if (!validator.matcher(column.name).matches()) {
+				throw new IllegalArgumentException("Invalid column name [ " + column.name + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+			}
+		}
+	}
+
+	public static void validateTableName(String tableName) {
+		if (!validator.matcher(tableName).matches()) {
+			throw new IllegalArgumentException("Invalid table name [ " + tableName + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+		}
+	}
+
+	public static void validateShardName(String shardName) {
+		if (!validator.matcher(shardName).matches()) {
+			throw new IllegalArgumentException("Invalid shard name [ " + shardName + " ]. It should contain only this pattern [A-Za-z0-9_-]");
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/Converter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/Converter.java b/blur-core/src/main/java/org/apache/blur/utils/Converter.java
new file mode 100644
index 0000000..cad0f45
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/Converter.java
@@ -0,0 +1,21 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+public interface Converter<F, T> {
+  T convert(F from) throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/ForkJoin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/ForkJoin.java b/blur-core/src/main/java/org/apache/blur/utils/ForkJoin.java
new file mode 100644
index 0000000..2ae88cc
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/ForkJoin.java
@@ -0,0 +1,92 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.utils.BlurExecutorCompletionService.Cancel;
+
+
+public class ForkJoin {
+
+  private static Log LOG = LogFactory.getLog(ForkJoin.class);
+
+  public static interface ParallelCall<INPUT, OUTPUT> {
+    OUTPUT call(INPUT input) throws Exception;
+  }
+
+  public static interface ParallelReturn<OUTPUT> {
+    OUTPUT merge(Merger<OUTPUT> merger) throws BlurException;
+  }
+
+  public static interface Merger<OUTPUT> {
+    OUTPUT merge(BlurExecutorCompletionService<OUTPUT> service) throws BlurException;
+  }
+
+  public static Cancel CANCEL = new Cancel() {
+    @Override
+    public void cancel() {
+      // do nothing
+    }
+  };
+
+  public static <INPUT, OUTPUT> ParallelReturn<OUTPUT> execute(ExecutorService executor, Iterable<INPUT> it, final ParallelCall<INPUT, OUTPUT> parallelCall) {
+    return execute(executor, it, parallelCall, CANCEL);
+  }
+
+  public static <INPUT, OUTPUT> ParallelReturn<OUTPUT> execute(ExecutorService executor, Iterable<INPUT> it, final ParallelCall<INPUT, OUTPUT> parallelCall, Cancel cancel) {
+    final BlurExecutorCompletionService<OUTPUT> service = new BlurExecutorCompletionService<OUTPUT>(executor, cancel);
+    for (final INPUT input : it) {
+      service.submit(new Callable<OUTPUT>() {
+        @Override
+        public OUTPUT call() throws Exception {
+          return parallelCall.call(input);
+        }
+      });
+    }
+    return new ParallelReturn<OUTPUT>() {
+      @Override
+      public OUTPUT merge(Merger<OUTPUT> merger) throws BlurException {
+        boolean exception = true;
+        try {
+          OUTPUT merge = merger.merge(service);
+          exception = false;
+          return merge;
+        } finally {
+          if (exception) {
+            service.cancelAll();
+          }
+        }
+      }
+    };
+  }
+
+  public static <T> T ignoreExecutionException(Future<T> future, T defaultValue) throws InterruptedException {
+    try {
+      return future.get();
+    } catch (ExecutionException e) {
+      LOG.error("Error while trying to execute task [{0}]", e, e.getMessage());
+      return defaultValue;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/IterableConverter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/IterableConverter.java b/blur-core/src/main/java/org/apache/blur/utils/IterableConverter.java
new file mode 100644
index 0000000..c6f22f1
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/IterableConverter.java
@@ -0,0 +1,36 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.Iterator;
+
+public class IterableConverter<F, T> implements Iterable<T> {
+
+  private Converter<F, T> converter;
+  private Iterable<F> iterable;
+
+  public IterableConverter(Iterable<F> iterable, Converter<F, T> converter) {
+    this.converter = converter;
+    this.iterable = iterable;
+  }
+
+  @Override
+  public Iterator<T> iterator() {
+    return new IteratorConverter<F, T>(iterable.iterator(), converter);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/IteratorConverter.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/IteratorConverter.java b/blur-core/src/main/java/org/apache/blur/utils/IteratorConverter.java
new file mode 100644
index 0000000..24d769e
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/IteratorConverter.java
@@ -0,0 +1,50 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.Iterator;
+
+public class IteratorConverter<F, T> implements Iterator<T> {
+
+  private Converter<F, T> converter;
+  private Iterator<F> iterator;
+
+  public IteratorConverter(Iterator<F> iterator, Converter<F, T> converter) {
+    this.converter = converter;
+    this.iterator = iterator;
+  }
+
+  @Override
+  public boolean hasNext() {
+    return iterator.hasNext();
+  }
+
+  @Override
+  public T next() {
+    try {
+      return converter.convert(iterator.next());
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public void remove() {
+    iterator.remove();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java b/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java
new file mode 100644
index 0000000..2c9a2c9
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/ObjectSize.java
@@ -0,0 +1,37 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.lang.instrument.Instrumentation;
+
+public class ObjectSize {
+
+  private static Instrumentation instrumentation;
+
+  public static void premain(String agentArgs, Instrumentation inst) {
+    instrumentation = inst;
+  }
+
+  public static long getSizeInBytes(Object o) {
+    if (instrumentation == null)
+      return -1;
+    if (o == null)
+      return 0;
+    return instrumentation.getObjectSize(o);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/PrimeDocCache.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/PrimeDocCache.java b/blur-core/src/main/java/org/apache/blur/utils/PrimeDocCache.java
new file mode 100644
index 0000000..f10502d
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/PrimeDocCache.java
@@ -0,0 +1,90 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.ReaderClosedListener;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.OpenBitSet;
+
+
+public class PrimeDocCache {
+
+  private static final Log LOG = LogFactory.getLog(PrimeDocCache.class);
+
+  public static final OpenBitSet EMPTY_BIT_SET = new OpenBitSet();
+
+  private static Map<Object, OpenBitSet> primeDocMap = new ConcurrentHashMap<Object, OpenBitSet>();
+
+  /**
+   * The way this method is called via warm up methods the likelihood of
+   * creating multiple bitsets during a race condition is very low, that's why
+   * this method is not synced.
+   */
+  public static OpenBitSet getPrimeDocBitSet(IndexReader reader) throws IOException {
+    Object key = reader.getCoreCacheKey();
+    OpenBitSet bitSet = primeDocMap.get(key);
+    if (bitSet == null) {
+      reader.addReaderClosedListener(new ReaderClosedListener() {
+        @Override
+        public void onClose(IndexReader reader) {
+          Object key = reader.getCoreCacheKey();
+          LOG.debug("Current size [" + primeDocMap.size() + "] Prime Doc BitSet removing for segment [" + reader + "]");
+          primeDocMap.remove(key);
+        }
+      });
+      LOG.debug("Prime Doc BitSet missing for segment [" + reader + "] current size [" + primeDocMap.size() + "]");
+      final OpenBitSet bs = new OpenBitSet(reader.maxDoc());
+      primeDocMap.put(key, bs);
+      IndexSearcher searcher = new IndexSearcher(reader);
+      searcher.search(new TermQuery(BlurConstants.PRIME_DOC_TERM), new Collector() {
+        
+        @Override
+        public void setScorer(Scorer scorer) throws IOException {
+          
+        }
+        
+        @Override
+        public void setNextReader(AtomicReaderContext atomicReaderContext) throws IOException {
+        }
+        
+        @Override
+        public void collect(int doc) throws IOException {
+          bs.set(doc);
+        }
+        
+        @Override
+        public boolean acceptsDocsOutOfOrder() {
+          return false;
+        }
+      });
+      return bs;
+    }
+    return bitSet;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java b/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java
new file mode 100644
index 0000000..7d5eff2
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/QueryCache.java
@@ -0,0 +1,107 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.SortedSet;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import com.googlecode.concurrentlinkedhashmap.EvictionListener;
+
+public class QueryCache implements EvictionListener<QueryCacheKey, QueryCacheEntry> {
+
+  private static final Log LOG = LogFactory.getLog(QueryCache.class);
+
+  private long _ttl;
+  private String _name;
+  private int _cachedElements;
+  private ConcurrentLinkedHashMap<QueryCacheKey, QueryCacheEntry> _cache;
+
+  public QueryCache(String name, int cachedElements, long ttl) {
+    _name = name;
+    _cachedElements = cachedElements;
+    _ttl = ttl;
+    _cache = new ConcurrentLinkedHashMap.Builder<QueryCacheKey, QueryCacheEntry>().maximumWeightedCapacity(_cachedElements).listener(this).build();
+  }
+
+  @Override
+  public void onEviction(QueryCacheKey key, QueryCacheEntry value) {
+    LOG.debug("Cache [" + _name + "] key [" + key + "] value [" + value + "] evicted.");
+  }
+
+  @SuppressWarnings("deprecation")
+  public static QueryCacheKey getNormalizedBlurQueryKey(String table, BlurQuery blurQuery) {
+    BlurQuery newBlurQuery = new BlurQuery(blurQuery);
+    newBlurQuery.allowStaleData = false;
+    newBlurQuery.useCacheIfPresent = false;
+    newBlurQuery.userContext = null;
+    newBlurQuery.maxQueryTime = 0;
+    newBlurQuery.uuid = 0;
+    newBlurQuery.startTime = 0;
+    newBlurQuery.modifyFileCaches = false;
+    return new QueryCacheKey(table, newBlurQuery);
+  }
+
+  public boolean isValid(QueryCacheEntry entry, SortedSet<String> currentShards) {
+    if (!isValid(entry)) {
+      return false;
+    }
+    if (!entry.shards.equals(currentShards)) {
+      return false;
+    }
+    return true;
+  }
+
+  public boolean isValid(QueryCacheEntry entry) {
+    if (entry == null) {
+      return false;
+    }
+    if (entry.timestamp + _ttl < System.currentTimeMillis()) {
+      return false;
+    }
+    return true;
+  }
+
+  public BlurResults cache(String table, BlurQuery original, BlurResults results) {
+    if (results == null) {
+      return null;
+    }
+    if (original != null && original.cacheResult) {
+      LOG.debug("Caching results for query [{0}]", original);
+      BlurResults cacheResults = new BlurResults(results);
+      cacheResults.query = null;
+      put(getNormalizedBlurQueryKey(table, original), new QueryCacheEntry(cacheResults));
+    }
+    return results;
+  }
+
+  public void put(QueryCacheKey key, QueryCacheEntry value) {
+    _cache.put(key, value);
+  }
+
+  public QueryCacheEntry get(QueryCacheKey key) {
+    return _cache.get(key);
+  }
+
+  public void remove(QueryCacheKey key) {
+    _cache.remove(key);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java b/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java
new file mode 100644
index 0000000..8b66f46
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/QueryCacheEntry.java
@@ -0,0 +1,46 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResults;
+
+
+public class QueryCacheEntry {
+  public BlurResults results;
+  public long timestamp;
+  public SortedSet<String> shards;
+
+  public QueryCacheEntry(BlurResults cacheResults) {
+    results = cacheResults;
+    timestamp = System.currentTimeMillis();
+    if (cacheResults != null && cacheResults.shardInfo != null) {
+      shards = new TreeSet<String>(cacheResults.shardInfo.keySet());
+    } else {
+      shards = new TreeSet<String>();
+    }
+  }
+
+  public BlurResults getBlurResults(BlurQuery blurQuery) {
+    BlurResults blurResults = new BlurResults(results);
+    blurResults.query = blurQuery;
+    return blurResults;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java b/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java
new file mode 100644
index 0000000..e17aa23
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/QueryCacheKey.java
@@ -0,0 +1,77 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.blur.thrift.generated.BlurQuery;
+
+public class QueryCacheKey {
+  private String table;
+  private BlurQuery query;
+
+  public String getTable() {
+    return table;
+  }
+
+  public void setTable(String table) {
+    this.table = table;
+  }
+
+  public BlurQuery getQuery() {
+    return query;
+  }
+
+  public void setQuery(BlurQuery query) {
+    this.query = query;
+  }
+
+  public QueryCacheKey(String table, BlurQuery query) {
+    this.table = table;
+    this.query = query;
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((query == null) ? 0 : query.hashCode());
+    result = prime * result + ((table == null) ? 0 : table.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    QueryCacheKey other = (QueryCacheKey) obj;
+    if (query == null) {
+      if (other.query != null)
+        return false;
+    } else if (!query.equals(other.query))
+      return false;
+    if (table == null) {
+      if (other.table != null)
+        return false;
+    } else if (!table.equals(other.table))
+      return false;
+    return true;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java b/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java
new file mode 100644
index 0000000..fb31401
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/ReaderBlurRecord.java
@@ -0,0 +1,29 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+public interface ReaderBlurRecord {
+
+  void setRecordIdStr(String value);
+
+  void addColumn(String name, String value);
+
+  void setFamilyStr(String family);
+
+  void setRowIdStr(String rowId);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java b/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java
new file mode 100644
index 0000000..e537e2f
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/ReferenceIterable.java
@@ -0,0 +1,55 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+public class ReferenceIterable<T> implements Iterable<T> {
+
+  private T t;
+
+  public ReferenceIterable(T t) {
+    this.t = t;
+  }
+
+  @Override
+  public Iterator<T> iterator() {
+    return new Iterator<T>() {
+      boolean taken = false;
+
+      @Override
+      public boolean hasNext() {
+        return !taken;
+      }
+
+      @Override
+      public T next() {
+        if (taken) {
+          throw new NoSuchElementException();
+        }
+        taken = true;
+        return t;
+      }
+
+      @Override
+      public void remove() {
+        throw new UnsupportedOperationException();
+      }
+    };
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/ResetableDocumentStoredFieldVisitor.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/ResetableDocumentStoredFieldVisitor.java b/blur-core/src/main/java/org/apache/blur/utils/ResetableDocumentStoredFieldVisitor.java
new file mode 100644
index 0000000..24fee3e
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/ResetableDocumentStoredFieldVisitor.java
@@ -0,0 +1,118 @@
+package org.apache.blur.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFieldVisitor;
+
+/** A {@link StoredFieldVisitor} that creates a {@link
+ *  Document} containing all stored fields, or only specific
+ *  requested fields provided to {@link #DocumentStoredFieldVisitor(Set)}.
+ *  <p>
+ *  This is used by {@link IndexReader#document(int)} to load a
+ *  document.
+ *
+ * @lucene.experimental */
+
+public class ResetableDocumentStoredFieldVisitor extends StoredFieldVisitor {
+  private Document doc = new Document();
+  private final Set<String> fieldsToAdd;
+
+  /** Load only fields named in the provided <code>Set&lt;String&gt;</code>. */
+  public ResetableDocumentStoredFieldVisitor(Set<String> fieldsToAdd) {
+    this.fieldsToAdd = fieldsToAdd;
+  }
+
+  /** Load only fields named in the provided <code>Set&lt;String&gt;</code>. */
+  public ResetableDocumentStoredFieldVisitor(String... fields) {
+    fieldsToAdd = new HashSet<String>(fields.length);
+    for(String field : fields) {
+      fieldsToAdd.add(field);
+    }
+  }
+
+  /** Load all stored fields. */
+  public ResetableDocumentStoredFieldVisitor() {
+    this.fieldsToAdd = null;
+  }
+
+  @Override
+  public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
+    doc.add(new StoredField(fieldInfo.name, value));
+  }
+
+  @Override
+  public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+    final FieldType ft = new FieldType(TextField.TYPE_STORED);
+    ft.setStoreTermVectors(fieldInfo.hasVectors());
+    ft.setIndexed(fieldInfo.isIndexed());
+    ft.setOmitNorms(fieldInfo.omitsNorms());
+    ft.setIndexOptions(fieldInfo.getIndexOptions());
+    doc.add(new Field(fieldInfo.name, value, ft));
+  }
+
+  @Override
+  public void intField(FieldInfo fieldInfo, int value) {
+    doc.add(new StoredField(fieldInfo.name, value));
+  }
+
+  @Override
+  public void longField(FieldInfo fieldInfo, long value) {
+    doc.add(new StoredField(fieldInfo.name, value));
+  }
+
+  @Override
+  public void floatField(FieldInfo fieldInfo, float value) {
+    doc.add(new StoredField(fieldInfo.name, value));
+  }
+
+  @Override
+  public void doubleField(FieldInfo fieldInfo, double value) {
+    doc.add(new StoredField(fieldInfo.name, value));
+  }
+
+  @Override
+  public Status needsField(FieldInfo fieldInfo) throws IOException {
+    return fieldsToAdd == null || fieldsToAdd.contains(fieldInfo.name) ? Status.YES : Status.NO;
+  }
+
+  /**
+   * Retrieve the visited document.
+   * @return Document populated with stored fields. Note that only
+   *         the stored information in the field instances is valid,
+   *         data such as boosts, indexing options, term vector options,
+   *         etc is not set.
+   */
+  public Document getDocument() {
+    return doc;
+  }
+  
+  public void reset() {
+    doc = new Document();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java b/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
new file mode 100644
index 0000000..4f09fe2
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/RowDocumentUtil.java
@@ -0,0 +1,91 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import static org.apache.blur.utils.BlurConstants.*;
+import static org.apache.blur.utils.BlurConstants.ROW_ID;
+import static org.apache.blur.utils.BlurConstants.SEP;
+
+import java.util.ArrayList;
+
+import org.apache.blur.thrift.generated.FetchRecordResult;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.Row;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexableField;
+
+
+public class RowDocumentUtil {
+
+  public static FetchRecordResult getRecord(Document document) {
+    FetchRecordResult result = new FetchRecordResult();
+    BlurThriftRecord record = new BlurThriftRecord();
+    String rowId = readRecord(document, record);
+    result.setRecord(record);
+    result.setRowid(rowId);
+    return result;
+  }
+
+  public static Row getRow(Iterable<Document> docs) {
+    Row row = new Row();
+    boolean empty = true;
+    if (docs == null) {
+      return null;
+    }
+    for (Document document : docs) {
+      empty = false;
+      BlurThriftRecord record = new BlurThriftRecord();
+      String rowId = readRecord(document, record);
+      if (record.getColumns() != null) {
+        row.addToRecords(record);
+      }
+      if (row.id == null) {
+        row.setId(rowId);
+      }
+      row.recordCount++;
+    }
+    if (empty) {
+      return null;
+    }
+    if (row.records == null) {
+      row.records = new ArrayList<Record>();
+    }
+    return row;
+  }
+
+  public static String readRecord(Document document, ReaderBlurRecord reader) {
+    String rowId = null;
+    for (IndexableField field : document.getFields()) {
+      if (field.name().equals(ROW_ID)) {
+        rowId = field.stringValue();
+      } else if (field.name().equals(RECORD_ID)) {
+        reader.setRecordIdStr(field.stringValue());
+      } else if (field.name().equals(FAMILY)) {
+        reader.setFamilyStr(field.stringValue());
+      } else {
+        String name = field.name();
+        int index = name.indexOf(SEP);
+        if (index < 0) {
+          continue;
+        }
+        name = name.substring(index + 1);
+        reader.addColumn(name, field.stringValue());
+      }
+    }
+    return rowId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java b/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java
new file mode 100644
index 0000000..f45be2b
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/TableShardCountCollapser.java
@@ -0,0 +1,164 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.blur.store.hdfs.HdfsDirectory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Version;
+
+/**
+ * This class is used to reduce the total number of shards of a table. The main
+ * use would be if during an indexing job the number of reducers were increased
+ * to make indexing faster, but the total number of shards in the table needed
+ * to be smaller. This utility safely collapses indexes together thus reducing
+ * the total number of shards in the table.
+ * 
+ * For example if you wanted to run 1024 reducers but only wanted to run 128
+ * shards in a table. After the bulk map reducer job finishes, this utility
+ * could be executed:
+ * 
+ * TableShardCountCollapser <hdfs path> 128
+ * 
+ * The result would be 128 shards in the table path.
+ * 
+ */
+public class TableShardCountCollapser extends Configured implements Tool {
+
+  public static void main(String[] args) throws Exception {
+    // Let ToolRunner handle generic command-line options
+    int res = ToolRunner.run(new Configuration(), new TableShardCountCollapser(), args);
+    System.exit(res);
+  }
+
+  private Path path;
+
+  @Override
+  public int run(String[] args) throws Exception {
+    // prompt to make sure the table is not enabled
+
+    Path path = new Path(args[0]);
+    int count = Integer.parseInt(args[1]);
+    setTablePath(path);
+    collapseShardsTo(count);
+    return 0;
+  }
+
+  public boolean validateCount(int count) throws IOException {
+    if (getCollapsePossibilities().contains(count)) {
+      return true;
+    }
+    return false;
+  }
+
+  public void setTablePath(Path path) {
+    this.path = path;
+  }
+
+  public List<Integer> getCollapsePossibilities() throws IOException {
+    FileSystem fileSystem = path.getFileSystem(getConf());
+    FileStatus[] listStatus = fileSystem.listStatus(path);
+    SortedSet<String> shards = new TreeSet<String>();
+    for (FileStatus status : listStatus) {
+      Path shardPath = status.getPath();
+      if (shardPath.getName().startsWith(BlurConstants.SHARD_PREFIX)) {
+        shards.add(shardPath.getName());
+      }
+    }
+    validateShards(shards);
+    List<Integer> result = getFactors(shards.size());
+    return result;
+  }
+
+  private List<Integer> getFactors(int size) {
+    List<Integer> result = new ArrayList<Integer>();
+    for (int i = 1; i < size; i++) {
+      if (size % i == 0) {
+        result.add(i);
+      }
+    }
+    return result;
+  }
+
+  private void validateShards(SortedSet<String> shards) {
+    int count = shards.size();
+    for (int i = 0; i < count; i++) {
+      if (!shards.contains(BlurUtil.getShardName(i))) {
+        throw new RuntimeException("Invalid table");
+      }
+    }
+  }
+
+  public void collapseShardsTo(int newShardCount) throws IOException {
+    if (!validateCount(newShardCount)) {
+      throw new RuntimeException("Count [" + newShardCount + "] is not valid, valid values are ["
+          + getCollapsePossibilities() + "]");
+    }
+
+    Path[] paths = getPaths();
+    int numberOfShardsToMergePerPass = paths.length / newShardCount;
+    for (int i = 0; i < newShardCount; i++) {
+      System.out.println("Base Index [" + paths[i] + "]");
+      IndexWriterConfig lconf = new IndexWriterConfig(Version.LUCENE_42, new KeywordAnalyzer());
+      HdfsDirectory dir = new HdfsDirectory(getConf(), paths[i]);
+      IndexWriter indexWriter = new IndexWriter(dir, lconf);
+      Directory[] dirs = new Directory[numberOfShardsToMergePerPass - 1];
+      Path[] pathsToDelete = new Path[numberOfShardsToMergePerPass - 1];
+      for (int p = 1; p < numberOfShardsToMergePerPass; p++) {
+        Path pathToMerge = paths[i + p * newShardCount];
+        System.out.println("Merge [" + pathToMerge + "]");
+        dirs[p - 1] = new HdfsDirectory(getConf(), pathToMerge);
+        pathsToDelete[p - 1] = pathToMerge;
+      }
+      indexWriter.addIndexes(dirs);
+      indexWriter.close();
+      FileSystem fileSystem = path.getFileSystem(getConf());
+      for (Path p : pathsToDelete) {
+        fileSystem.delete(p, true);
+      }
+    }
+  }
+
+  private Path[] getPaths() throws IOException {
+    FileSystem fileSystem = path.getFileSystem(getConf());
+    FileStatus[] listStatus = fileSystem.listStatus(path);
+    SortedSet<Path> shards = new TreeSet<Path>();
+    for (FileStatus status : listStatus) {
+      Path shardPath = status.getPath();
+      if (shardPath.getName().startsWith(BlurConstants.SHARD_PREFIX)) {
+        shards.add(shardPath);
+      }
+    }
+    return shards.toArray(new Path[shards.size()]);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java b/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
new file mode 100644
index 0000000..e47cba8
--- /dev/null
+++ b/blur-core/src/main/java/org/apache/blur/utils/TermDocIterable.java
@@ -0,0 +1,101 @@
+package org.apache.blur.utils;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.Bits;
+
+public class TermDocIterable implements Iterable<Document> {
+
+  private DocsEnum docsEnum;
+  private AtomicReader reader;
+  private ResetableDocumentStoredFieldVisitor fieldSelector;
+
+  public TermDocIterable(DocsEnum docsEnum, AtomicReader reader) {
+    this(docsEnum, reader, new ResetableDocumentStoredFieldVisitor());
+  }
+
+  public TermDocIterable(DocsEnum docsEnum, AtomicReader reader, ResetableDocumentStoredFieldVisitor fieldSelector) {
+    if (docsEnum == null) {
+      throw new NullPointerException("docsEnum can not be null.");
+    }
+    this.docsEnum = docsEnum;
+    this.reader = reader;
+    this.fieldSelector = fieldSelector;
+  }
+
+  @Override
+  public Iterator<Document> iterator() {
+    return new Iterator<Document>() {
+      private boolean hasNext = getNext();
+
+      @Override
+      public boolean hasNext() {
+        return hasNext;
+      }
+
+      @Override
+      public Document next() {
+        Document doc;
+        try {
+          doc = getDoc();
+          hasNext = getNext();
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+        return doc;
+      }
+
+      @Override
+      public void remove() {
+
+      }
+    };
+  }
+
+  private Document getDoc() throws IOException {
+    fieldSelector.reset();
+    reader.document(docsEnum.docID(), fieldSelector);
+    return fieldSelector.getDocument();
+  }
+
+  private boolean getNext() {
+    try {
+      int next = docsEnum.nextDoc();
+      if (next == DocIdSetIterator.NO_MORE_DOCS) {
+        return false;
+      }
+      Bits liveDocs = MultiFields.getLiveDocs(reader);
+      if (liveDocs != null) {
+        while (!liveDocs.get(docsEnum.docID())) {
+          next = docsEnum.nextDoc();
+        }
+      }
+      return next == DocIdSetIterator.NO_MORE_DOCS ? false : true;
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-core/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/blur-core/src/main/resources/log4j.properties b/blur-core/src/main/resources/log4j.properties
new file mode 100644
index 0000000..56dc046
--- /dev/null
+++ b/blur-core/src/main/resources/log4j.properties
@@ -0,0 +1,51 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+
+# Define some default values that can be overridden by system properties
+blur.root.logger=INFO,console
+blur.log.dir=.
+blur.log.file=blur.log
+
+# Define the root logger to the system property "blur.root.logger".
+log4j.rootLogger=${blur.root.logger}
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${blur.log.dir}/${blur.log.file}
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Console
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO


Mime
View raw message