incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [27/92] [abbrv] [partial] Fixed BLUR-126.
Date Tue, 11 Jun 2013 02:41:13 GMT
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-query/src/test/java/org/apache/blur/search/TestingPagingCollector.java
----------------------------------------------------------------------
diff --git a/blur-query/src/test/java/org/apache/blur/search/TestingPagingCollector.java b/blur-query/src/test/java/org/apache/blur/search/TestingPagingCollector.java
new file mode 100644
index 0000000..5eaad3f
--- /dev/null
+++ b/blur-query/src/test/java/org/apache/blur/search/TestingPagingCollector.java
@@ -0,0 +1,84 @@
+package org.apache.blur.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.blur.lucene.search.IterablePaging;
+import org.apache.blur.lucene.search.IterablePaging.ProgressRef;
+import org.apache.blur.lucene.search.IterablePaging.TotalHitsRef;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.RAMDirectory;
+import org.junit.Test;
+
+/**
+ * Testing the paging collector.
+ * 
+ * @author Aaron McCurry
+ */
+public class TestingPagingCollector {
+
+  @Test
+  public void testNothingYet() {
+
+  }
+
+  public static void main(String[] args) throws Exception {
+    IndexReader reader = getReaderFlatScore(13245);
+    IndexSearcher searcher = new IndexSearcher(reader);
+
+    TotalHitsRef totalHitsRef = new TotalHitsRef();
+    ProgressRef progressRef = new ProgressRef();
+
+    TermQuery query = new TermQuery(new Term("f1", "value"));
+    IterablePaging paging = new IterablePaging(new AtomicBoolean(true), searcher, query, 100, null, null);
+
+    for (ScoreDoc sd : paging.skipTo(90).gather(20).totalHits(totalHitsRef).progress(progressRef)) {
+
+      System.out.println("time [" + progressRef.queryTime() + "] " + "total hits [" + totalHitsRef.totalHits() + "] "
+          + "searches [" + progressRef.searchesPerformed() + "] " + "position [" + progressRef.currentHitPosition()
+          + "] " + "doc id [" + sd.doc + "] " + "score [" + sd.score + "]");
+    }
+  }
+
+  private static IndexReader getReaderFlatScore(int length) throws Exception {
+    RAMDirectory directory = new RAMDirectory();
+    IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer()));
+    for (int i = 0; i < length; i++) {
+      Document document = new Document();
+      document.add(new Field("f1", "value", Store.NO, Index.ANALYZED_NO_NORMS));
+      indexWriter.addDocument(document);
+    }
+    indexWriter.close();
+    return IndexReader.open(directory);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/README.textile
----------------------------------------------------------------------
diff --git a/blur-shell/README.textile b/blur-shell/README.textile
new file mode 100644
index 0000000..d7264cb
--- /dev/null
+++ b/blur-shell/README.textile
@@ -0,0 +1,16 @@
+h1. Blur shell
+
+A jline based shell which can attach to a Blur server.
+
+h2. Usage
+
+Currently the easiest way to run is 
+
+mvn exec:java -Dexec.mainClass="org.apache.blur.shell.Main" -Dexec.args="host:port"
+
+# create a table that spans three shards
+createtable hdfs://blurhost:9000/blur/tables/table1 table1 3
+
+# index an accesslog
+indexaccesslog /tmp/access_log 100 table1 \\[(.*)\\]\\s+(\\d+)\\s+([\\d\\.]+)\\s+([^\\s]+)\\s+([\\d\\.]+)\\s+([^\\s]+)\\s+([^\\s]+)\\s+([^\\s]+)\\s* date bytes1 sourceip status bytes2 operation url type
+

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/pom.xml
----------------------------------------------------------------------
diff --git a/blur-shell/pom.xml b/blur-shell/pom.xml
new file mode 100644
index 0000000..37f90eb
--- /dev/null
+++ b/blur-shell/pom.xml
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.apache.blur</groupId>
+		<artifactId>blur</artifactId>
+		<version>0.1.5</version>
+        <relativePath>../pom.xml</relativePath>
+	</parent>
+	<groupId>org.apache.blur</groupId>
+	<artifactId>blur-shell</artifactId>
+	<packaging>jar</packaging>
+	<name>Blur Shell</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.blur</groupId>
+            <artifactId>blur-thrift</artifactId>
+            <version>${project.version}</version>
+
+            <exclusions>
+                <exclusion>
+                    <groupId>javax.mail</groupId>
+                    <artifactId>mail</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.jms</groupId>
+                    <artifactId>jms</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jdmk</groupId>
+                    <artifactId>jmxtools</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jmx</groupId>
+                    <artifactId>jmxri</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>jline</groupId>
+            <artifactId>jline</artifactId>
+            <version>${jline.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+        	<groupId>com.google.guava</groupId>
+        	<artifactId>guava</artifactId>
+        	<version>${guava.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <finalName>blur-shell-${project.version}</finalName>
+
+        <plugins>
+		  <plugin>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-compiler-plugin</artifactId>
+			<configuration>
+			  <source>1.6</source>
+			  <target>1.6</target>
+			</configuration>
+		  </plugin>
+
+          <plugin>
+            <artifactId>maven-assembly-plugin</artifactId>
+            <version>2.3</version>
+            <configuration>
+              <descriptorRefs>
+                <descriptorRef>jar-with-dependencies</descriptorRef>
+              </descriptorRefs>
+              <archive>
+                <manifest>
+                  <mainClass>org.apache.blur.shell.Main</mainClass>
+                </manifest>
+              </archive>
+            </configuration>
+            <executions>
+              <execution>
+                <id>make-assembly</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>single</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+    </build>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/blur-shell/src/assemble/bin.xml b/blur-shell/src/assemble/bin.xml
new file mode 100644
index 0000000..463fdbc
--- /dev/null
+++ b/blur-shell/src/assemble/bin.xml
@@ -0,0 +1,8 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+  <id>bin</id>
+  <formats>
+    <format>jar</format>
+  </formats>
+</assembly>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/Command.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/Command.java b/blur-shell/src/main/java/org/apache/blur/shell/Command.java
new file mode 100644
index 0000000..36da1ec
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/Command.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Blur.Client;
+
+public abstract class Command {
+  @SuppressWarnings("serial")
+  public static class CommandException extends Exception {
+    public CommandException(String msg) {
+      super(msg);
+    }
+  }
+  abstract public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException;
+  abstract public String help();
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/ControllersEchoCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/ControllersEchoCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/ControllersEchoCommand.java
new file mode 100644
index 0000000..5d82874
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/ControllersEchoCommand.java
@@ -0,0 +1,61 @@
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.List;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class ControllersEchoCommand extends Command {
+
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args) throws CommandException, TException, BlurException {
+    List<String> controllerServerList = client.controllerServerList();
+    String nodeName = getNodeName();
+    for (String controller : controllerServerList) {
+      if (isSameServer(controller, nodeName)) {
+        out.println(controller + "*");
+      } else {
+        out.println(controller);
+      }
+    }
+  }
+
+  private boolean isSameServer(String controller, String nodeName) {
+    if (nodeName == null || controller == null) {
+      return false;
+    } else {
+      int i = controller.lastIndexOf(':');
+      if (i < 0) {
+        return false;
+      }
+      if (nodeName.equals(controller.substring(0, i))) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @Override
+  public String help() {
+    return "list controllers";
+  }
+
+  public static String getNodeName() {
+    try {
+      return InetAddress.getLocalHost().getHostName();
+    } catch (UnknownHostException e) {
+      String message = e.getMessage();
+      int index = message.indexOf(':');
+      if (index < 0) {
+        return null;
+      }
+      String nodeName = message.substring(0, index);
+      return nodeName;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/CreateTableCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/CreateTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/CreateTableCommand.java
new file mode 100644
index 0000000..a22b427
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/CreateTableCommand.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.AnalyzerDefinition;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.TableDescriptor;
+
+public class CreateTableCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 4) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tableuri = args[1];
+    String tablename = args[2];
+    int shardCount = Integer.parseInt(args[3]);
+
+    AnalyzerDefinition ad = new AnalyzerDefinition();
+
+    TableDescriptor td = new TableDescriptor(); 
+    td.setTableUri(tableuri);
+    td.setCluster("default");
+    td.setAnalyzerDefinition(ad);
+    td.setName(tablename);
+
+    td.setShardCount(shardCount);
+
+    if (Main.debug) {
+      out.println(td.toString());
+      out.flush();
+    }
+
+    client.createTable(td);
+  }
+
+  @Override
+  public String help() {
+    return "create the named table, args; tableuri tablename shardcount";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/DescribeTableCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/DescribeTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/DescribeTableCommand.java
new file mode 100644
index 0000000..15e9a5b
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/DescribeTableCommand.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class DescribeTableCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 2) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+
+    out.println(client.describe(tablename));
+  }
+
+  @Override
+  public String help() {
+    return "describe the named table, args; tablename";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/EnableDisableTableCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/EnableDisableTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/EnableDisableTableCommand.java
new file mode 100644
index 0000000..d3ab60e
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/EnableDisableTableCommand.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class EnableDisableTableCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 2) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+
+    if (args[0].equalsIgnoreCase("enabletable")) {
+      client.enableTable(tablename);
+    } else if (args[0].equalsIgnoreCase("disabletable")) {
+      client.disableTable(tablename);
+    } else {
+      throw new CommandException("unknown command" + args[0]);
+    }
+  }
+
+  @Override
+  public String help() {
+    return "change status of the named table, args; tablename";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java
new file mode 100644
index 0000000..287fe4e
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/GetRowCommand.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.FetchResult;
+import org.apache.blur.thrift.generated.FetchRowResult;
+import org.apache.blur.thrift.generated.Row;
+import org.apache.blur.thrift.generated.Selector;
+
+public class GetRowCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args) throws CommandException, TException, BlurException {
+    if (args.length != 3) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+    String rowId = args[2];
+
+    Selector selector = new Selector();
+    selector.setRowId(rowId);
+    FetchResult fetchRow = client.fetchRow(tablename, selector);
+    FetchRowResult rowResult = fetchRow.getRowResult();
+    if (rowResult == null) {
+      out.println("Row [" + rowId + "] not found.");
+      return;
+    }
+    Row row = rowResult.getRow();
+    if (row == null) {
+      out.println("Row [" + rowId + "] not found.");
+      return;
+    }
+    out.println(row);
+  }
+
+  @Override
+  public String help() {
+    return "display the specified row, args; tablename query";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/IndexAccessLogCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/IndexAccessLogCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/IndexAccessLogCommand.java
new file mode 100644
index 0000000..10c434b
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/IndexAccessLogCommand.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.LineNumberReader;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+
+import com.google.common.collect.Lists;
+
+public class IndexAccessLogCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length < 6) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    File logfile = new File(args[1]);
+    int batchSize = Integer.parseInt(args[2]);
+    String tablename = args[3];
+    String regex = args[4];
+
+    if (Main.debug) {
+      out.println(regex);
+    }
+
+    Pattern p = Pattern.compile(regex);
+    
+    try {
+      LineNumberReader reader = new LineNumberReader(new FileReader(logfile));
+      try {
+        String line;
+        List<RowMutation> mutations = Lists.newArrayList();
+        while((line = reader.readLine()) != null) {
+          Matcher m = p.matcher(line);
+          if (!m.matches()) {
+            continue;
+          }
+
+          List<Column> columns = new ArrayList<Column>();
+          for (int i = 5; i < args.length; i++) {
+            columns.add(new Column(args[i], m.group(i - 4)));
+          }
+
+          Record record = new Record();
+          record.setRecordId(UUID.randomUUID().toString());
+          record.setFamily("cf1");
+          record.setColumns(columns);
+
+          RecordMutation recordMutation = new RecordMutation();
+          recordMutation.setRecord(record);
+          recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+
+          List<RecordMutation> recordMutations = new ArrayList<RecordMutation>();
+          recordMutations.add(recordMutation);
+
+          RowMutation mutation = new RowMutation();
+          mutation.setTable(tablename);
+          mutation.setRowId(UUID.randomUUID().toString());
+          mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+          mutation.setRecordMutations(recordMutations);
+
+          mutations.add(mutation);
+
+          if (mutations.size() == batchSize) {
+            client.mutateBatch(mutations);
+            mutations.clear();
+          }
+        }
+        if (mutations.size() > 0) {
+          client.mutateBatch(mutations);
+          mutations.clear();
+        }
+      } finally {
+        reader.close();
+      }
+    } catch (IOException e) {
+      throw new CommandException(e.getMessage());
+    }
+  }
+
+  @Override
+  public String help() {
+    return "index an access log, args; file batchsize tablename regex colnames+";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/ListTablesCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/ListTablesCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/ListTablesCommand.java
new file mode 100644
index 0000000..06de2dd
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/ListTablesCommand.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class ListTablesCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    out.println("tables: " + client.tableList());
+  }
+
+  @Override
+  public String help() {
+    return "list tables";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/Main.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/Main.java b/blur-shell/src/main/java/org/apache/blur/shell/Main.java
new file mode 100644
index 0000000..b74a766
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/Main.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import jline.console.ConsoleReader;
+import jline.console.completer.Completer;
+import jline.console.completer.FileNameCompleter;
+import jline.console.completer.StringsCompleter;
+
+import org.apache.blur.shell.Command.CommandException;
+import org.apache.blur.shell.Main.QuitCommand.QuitCommandException;
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TBinaryProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TFramedTransport;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TSocket;
+import org.apache.blur.thirdparty.thrift_0_9_0.transport.TTransport;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+import com.google.common.collect.ImmutableMap;
+
+public class Main {
+  /** is debugging enabled - off by default */
+  static boolean debug = false;
+  /** is timing enabled - off by default */
+  static boolean timed = false;
+
+  private static Map<String, Command> commands;
+  
+  public static void usage() {
+    System.out.println("Usage: java " + Main.class.getName()
+        + " controller:port");
+  }
+
+  private static class DebugCommand extends Command {
+
+    @Override
+    public void doit(PrintWriter out, Client client, String[] args)
+        throws CommandException, TException, BlurException {
+      if (debug == true) {
+        debug = false;
+      } else {
+        debug = true;
+      }
+      out.println("debugging is now " + (debug ? "on" : "off"));
+    }
+
+    @Override
+    public String help() {
+      return "toggle debugging on/off";
+    }
+    
+  }
+
+  private static class TimedCommand extends Command {
+
+    @Override
+    public void doit(PrintWriter out, Client client, String[] args)
+        throws CommandException, TException, BlurException {
+      if (timed == true) {
+        timed = false;
+      } else {
+        timed = true;
+      }
+      out.println("timing of commands is now " + (timed ? "on" : "off"));
+    }
+
+    @Override
+    public String help() {
+      return "toggle timing of commands on/off";
+    }
+    
+  }
+
+  private static class HelpCommand extends Command {
+    @Override
+    public void doit(PrintWriter out, Client client, String[] args)
+        throws CommandException, TException, BlurException {
+      out.println("Available commands:");
+      for (Entry<String, Command> e: commands.entrySet()) {
+        out.println("  " + e.getKey() + " - " + e.getValue().help());
+      }
+    }
+
+    @Override
+    public String help() {
+      return "display help";
+    }
+  }
+
+  public static class QuitCommand extends Command {
+    @SuppressWarnings("serial")
+    public static class QuitCommandException extends CommandException {
+      public QuitCommandException() {
+        super("quit");
+      }
+    }
+
+    @Override
+    public void doit(PrintWriter out, Client client, String[] args)
+        throws CommandException, TException, BlurException {
+      throw new QuitCommandException();
+    }
+
+    @Override
+    public String help() {
+      return "exit the shell";
+    }
+  }
+
+  public static void main(String[] args) throws Throwable {
+    commands = new ImmutableMap.Builder<String,Command>()
+        .put("help", new HelpCommand())
+        .put("debug", new DebugCommand())
+        .put("timed", new TimedCommand())
+        .put("quit", new QuitCommand())
+        .put("listtables", new ListTablesCommand())
+        .put("createtable", new CreateTableCommand())
+        .put("enabletable", new EnableDisableTableCommand())
+        .put("disabletable", new EnableDisableTableCommand())
+        .put("removetable", new RemoveTableCommand())
+        .put("describetable", new DescribeTableCommand())
+        .put("tablestats", new TableStatsCommand())
+        .put("schema", new SchemaTableCommand())
+        .put("query", new QueryCommand())
+        .put("getrow", new GetRowCommand())
+        .put("mutaterow", new MutateRowCommand())
+        .put("indexaccesslog", new IndexAccessLogCommand())
+        .put("shardclusterlist", new ShardClusterListCommand())
+        .put("shardserverlayout", new ShardServerLayoutCommand())
+        .put("controllers", new ControllersEchoCommand())
+        .build();
+
+    try {
+      ConsoleReader reader = new ConsoleReader();
+
+      reader.setPrompt("blur> ");
+
+      if ((args == null) || (args.length != 1)) {
+        usage();
+        return;
+      }
+
+      String[] hostport = args[0].split(":"); 
+
+      if (hostport.length != 2) {
+        usage();
+        return;
+      }
+
+      List<Completer> completors = new LinkedList<Completer>();
+
+      completors.add(new StringsCompleter(commands.keySet()));
+      completors.add(new FileNameCompleter());
+
+      for (Completer c : completors) {
+        reader.addCompleter(c);
+      }
+
+      TTransport trans = new TSocket(hostport[0], Integer.parseInt(hostport[1]));
+      TProtocol proto = new TBinaryProtocol(new TFramedTransport(trans));
+      Client client = new Client(proto);
+      try {
+          trans.open();
+
+          String line;
+          PrintWriter out = new PrintWriter(reader.getOutput());
+          try {
+            while ((line = reader.readLine()) != null) {
+              line = line.trim();
+              // ignore empty lines and comments
+              if (line.length() == 0 || line.startsWith("#")) {
+                continue;
+              }
+              String[] commandArgs = line.split("\\s");
+              Command command = commands.get(commandArgs[0]);
+              if (command == null) {
+                out.println("unknown command \"" + commandArgs[0] + "\"");
+              } else {
+                long start = System.nanoTime();
+                try {
+                  command.doit(out, client, commandArgs);
+                } catch (QuitCommandException e) {
+                  // exit gracefully
+                  System.exit(0);
+                } catch (CommandException e) {
+                  out.println(e.getMessage());
+                  if (debug) {
+                    e.printStackTrace(out);
+                  }
+                } catch (BlurException e) {
+                  out.println(e.getMessage());
+                  if (debug) {
+                    e.printStackTrace(out);
+                  }
+                } finally {
+                  if (timed) {
+                    out.println("Last command took "
+                        + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)
+                        + "ms");
+                  }
+                }
+              }
+            }
+          } finally {
+            out.close();
+          }
+      } finally {
+          trans.close();
+      }
+    } catch (Throwable t) {
+      t.printStackTrace();
+      throw t;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/MutateRowCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/MutateRowCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/MutateRowCommand.java
new file mode 100644
index 0000000..86dcf05
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/MutateRowCommand.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+import org.apache.blur.thrift.generated.RecordMutation;
+import org.apache.blur.thrift.generated.RecordMutationType;
+import org.apache.blur.thrift.generated.RowMutation;
+import org.apache.blur.thrift.generated.RowMutationType;
+
+public class MutateRowCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 7) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+    String rowid = args[2];
+    String recordid = args[3];
+    String columnfamily = args[4];
+    String columnname = args[5];
+    String value = args[6];
+
+    List<Column> columns = new ArrayList<Column>();
+    columns.add(new Column(columnname, value));
+
+    Record record = new Record();
+    record.setRecordId(recordid);
+    record.setFamily(columnfamily);
+    record.setColumns(columns);
+
+    RecordMutation recordMutation = new RecordMutation();
+    recordMutation.setRecord(record);
+    recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
+
+    List<RecordMutation> recordMutations = new ArrayList<RecordMutation>();
+    recordMutations.add(recordMutation);
+
+    RowMutation mutation = new RowMutation();
+    mutation.setTable(tablename);
+    mutation.setRowId(rowid);
+    mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
+    mutation.setRecordMutations(recordMutations);
+
+    client.mutate(mutation);
+  }
+
+  @Override
+  public String help() {
+    return "mutate the specified row, args; tablename rowid recordid columnfamily columnname value";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java
new file mode 100644
index 0000000..5599e0b
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/QueryCommand.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurQuery;
+import org.apache.blur.thrift.generated.BlurResult;
+import org.apache.blur.thrift.generated.BlurResults;
+import org.apache.blur.thrift.generated.Selector;
+import org.apache.blur.thrift.generated.SimpleQuery;
+
+public class QueryCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length < 3) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+    String query = "";
+    for (int i = 2; i < args.length; i++) {
+      query += args[i] + " ";
+    }
+
+    BlurQuery blurQuery = new BlurQuery();
+    SimpleQuery simpleQuery = new SimpleQuery();
+    simpleQuery.setQueryStr(query);
+    blurQuery.setSimpleQuery(simpleQuery);
+    blurQuery.setSelector(new Selector());
+
+    if (Main.debug) {
+      out.println(blurQuery);
+    }
+
+    BlurResults blurResults = client.query(tablename, blurQuery);
+
+    if (Main.debug) {
+      out.println("shardinfo: " + blurResults.getShardInfo());
+    }
+
+    for (BlurResult result : blurResults.getResults()) {
+      out.println(result);
+    }
+  }
+
+  @Override
+  public String help() {
+    return "query the named table, args; tablename query";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/RemoveTableCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/RemoveTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/RemoveTableCommand.java
new file mode 100644
index 0000000..7690686
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/RemoveTableCommand.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class RemoveTableCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 2) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+
+    client.removeTable(tablename, true);
+  }
+
+  @Override
+  public String help() {
+    return "remove the named table, args; tablename";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java
new file mode 100644
index 0000000..a2609c2
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/SchemaTableCommand.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class SchemaTableCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 2) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+
+    out.println(client.schema(tablename));
+  }
+
+  @Override
+  public String help() {
+    return "schema of the named table, args; tablename";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/ShardClusterListCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/ShardClusterListCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/ShardClusterListCommand.java
new file mode 100644
index 0000000..3ccc6f3
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/ShardClusterListCommand.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class ShardClusterListCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 1) {
+      throw new CommandException("Invalid args: " + help());
+    }
+
+    out.println(client.shardClusterList());
+  }
+
+  @Override
+  public String help() {
+    return "list the clusters";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/ShardServerLayoutCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/ShardServerLayoutCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/ShardServerLayoutCommand.java
new file mode 100644
index 0000000..21e75ef
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/ShardServerLayoutCommand.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class ShardServerLayoutCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 2) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+
+    out.println(client.shardServerLayout(tablename));
+  }
+
+  @Override
+  public String help() {
+    return "list the server layout for a table, args: tablename";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/TableStatsCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/TableStatsCommand.java b/blur-shell/src/main/java/org/apache/blur/shell/TableStatsCommand.java
new file mode 100644
index 0000000..f8d7d18
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/TableStatsCommand.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.blur.shell;
+
+import java.io.PrintWriter;
+
+import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.Blur.Client;
+import org.apache.blur.thrift.generated.BlurException;
+
+public class TableStatsCommand extends Command {
+  @Override
+  public void doit(PrintWriter out, Client client, String[] args)
+      throws CommandException, TException, BlurException {
+    if (args.length != 2) {
+      throw new CommandException("Invalid args: " + help());
+    }
+    String tablename = args[1];
+
+    out.println(client.tableStats(tablename));
+  }
+
+  @Override
+  public String help() {
+    return "print stats for the named table";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-shell/src/main/java/org/apache/blur/shell/package-info.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/package-info.java b/blur-shell/src/main/java/org/apache/blur/shell/package-info.java
new file mode 100644
index 0000000..93727e0
--- /dev/null
+++ b/blur-shell/src/main/java/org/apache/blur/shell/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.blur.shell;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/pom.xml
----------------------------------------------------------------------
diff --git a/blur-store/pom.xml b/blur-store/pom.xml
new file mode 100644
index 0000000..a6702b5
--- /dev/null
+++ b/blur-store/pom.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>org.apache.blur</groupId>
+		<artifactId>blur</artifactId>
+		<version>0.1.5</version>
+        <relativePath>../pom.xml</relativePath>
+	</parent>
+	<groupId>org.apache.blur</groupId>
+	<artifactId>blur-store</artifactId>
+	<packaging>jar</packaging>
+	<name>Blur Store</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.blur</groupId>
+			<artifactId>blur-util</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-core</artifactId>
+			<version>${lucene.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-codecs</artifactId>
+			<version>${lucene.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-analyzers-common</artifactId>
+			<version>${lucene.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-queryparser</artifactId>
+			<version>${lucene.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>commons-cli</groupId>
+			<artifactId>commons-cli</artifactId>
+			<version>${commons-cli.version}</version>
+			<scope>compile</scope>
+		</dependency>
+		<dependency>
+			<groupId>com.googlecode.concurrentlinkedhashmap</groupId>
+			<artifactId>concurrentlinkedhashmap-lru</artifactId>
+			<version>${concurrentlinkedhashmap-lru.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>${log4j.version}</version>
+			<scope>provided</scope>
+			<exclusions>
+				<exclusion>
+					<groupId>javax.mail</groupId>
+					<artifactId>mail</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>javax.jms</groupId>
+					<artifactId>jms</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>com.sun.jdmk</groupId>
+					<artifactId>jmxtools</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>com.sun.jmx</groupId>
+					<artifactId>jmxri</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+	</dependencies>
+
+	<repositories>
+		<repository>
+			<id>libdir</id>
+			<url>file://${basedir}/../lib</url>
+		</repository>
+	</repositories>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<configuration>
+					<source>1.6</source>
+					<target>1.6</target>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<executions>
+					<execution>
+						<goals>
+							<goal>test-jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java b/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java
new file mode 100644
index 0000000..95f8b05
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/index/IndexWriter.java
@@ -0,0 +1,99 @@
+package org.apache.blur.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockObtainFailedException;
+
+public class IndexWriter extends org.apache.lucene.index.IndexWriter {
+
+  public static class LockOwnerException extends IOException {
+    private static final long serialVersionUID = -8211546713487754992L;
+
+    public LockOwnerException(String msg) {
+      super(msg);
+    }
+  }
+
+  private Lock internalLock;
+
+  public IndexWriter(Directory d, IndexWriterConfig conf) throws CorruptIndexException, LockObtainFailedException,
+      IOException {
+    super(d, conf);
+    try {
+      internalLock = getInternalLock();
+    } catch (Exception e) {
+      throw new RuntimeException("Could not get the write lock instance.", e);
+    }
+  }
+
+  private Lock getInternalLock() throws SecurityException, NoSuchFieldException, IllegalArgumentException,
+      IllegalAccessException {
+    Field field = org.apache.lucene.index.IndexWriter.class.getDeclaredField("writeLock");
+    field.setAccessible(true);
+    return (Lock) field.get(this);
+  }
+
+  @Override
+  protected void doAfterFlush() throws IOException {
+    super.doAfterFlush();
+    if (!internalLock.isLocked()) {
+      throw new LockOwnerException("Lock [" + internalLock + "] no longer has write lock.");
+    }
+  }
+
+  @Override
+  protected void doBeforeFlush() throws IOException {
+    super.doBeforeFlush();
+    if (!internalLock.isLocked()) {
+      throw new LockOwnerException("Lock [" + internalLock + "] no longer has write lock.");
+    }
+  }
+
+  /**
+   * This seems a little iffy, but basically only the writer instance itself can
+   * equal itself.
+   */
+  @Override
+  public boolean equals(Object obj) {
+    if (super.equals(obj)) {
+      return true;
+    } else if (obj == null) {
+      return false;
+    } else if (obj == this) {
+      return true;
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return super.hashCode();
+  }
+
+  @Override
+  public String toString() {
+    return "IndexWriter with directory [" + getDirectory() + "]";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java b/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
new file mode 100644
index 0000000..f6347bc
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/index/OptimisticLockFactory.java
@@ -0,0 +1,102 @@
+package org.apache.blur.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockFactory;
+
+public class OptimisticLockFactory extends LockFactory {
+
+  private final Configuration _configuration;
+  private final FileSystem _fileSystem;
+  private final String _baseLockKey;
+  private byte[] _lockKey;
+  private final Path _dir;
+
+  public OptimisticLockFactory(Configuration configuration, Path dir, String host, int pid) throws IOException {
+    _configuration = configuration;
+    _dir = dir;
+    _fileSystem = _dir.getFileSystem(_configuration);
+    _baseLockKey = host + "/" + pid;
+  }
+
+  @Override
+  public Lock makeLock(String lockName) {
+    final Path lockPath = new Path(_dir, lockName);
+    return new Lock() {
+      private boolean _set;
+
+      @Override
+      public boolean obtain() throws IOException {
+        if (_set) {
+          throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
+        }
+        try {
+          _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
+          FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
+          outputStream.write(_lockKey);
+          outputStream.close();
+        } finally {
+          _set = true;
+        }
+        return true;
+      }
+
+      @Override
+      public void release() throws IOException {
+        _fileSystem.delete(lockPath, false);
+      }
+
+      @Override
+      public boolean isLocked() throws IOException {
+        if (!_set) {
+          return false;
+        }
+        if (!_fileSystem.exists(lockPath)) {
+          return false;
+        }
+        FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
+        long len = fileStatus.getLen();
+        if (len != _lockKey.length) {
+          return false;
+        }
+        byte[] buf = new byte[_lockKey.length];
+        FSDataInputStream inputStream = _fileSystem.open(lockPath);
+        inputStream.readFully(buf);
+        inputStream.close();
+        if (Arrays.equals(_lockKey, buf)) {
+          return true;
+        }
+        return false;
+      }
+    };
+  }
+
+  @Override
+  public void clearLock(String lockName) throws IOException {
+    _fileSystem.delete(new Path(_dir, lockName), false);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java b/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
new file mode 100644
index 0000000..20ae366
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/LuceneVersionConstant.java
@@ -0,0 +1,28 @@
+package org.apache.blur.lucene;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.lucene.util.Version;
+
+public class LuceneVersionConstant {
+
+  /**
+   * All Lucene Version references should refer to this constant.
+   */
+  public static final Version LUCENE_VERSION = Version.LUCENE_43;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java
new file mode 100644
index 0000000..250cc69
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceCounter.java
@@ -0,0 +1,288 @@
+package org.apache.blur.lucene.store.refcounter;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.blur.store.hdfs.DirectoryDecorator;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.LockFactory;
+
+public class DirectoryReferenceCounter extends Directory implements DirectoryDecorator {
+
+  private final static Log LOG = LogFactory.getLog(DirectoryReferenceCounter.class);
+  private Directory directory;
+  private Map<String, AtomicInteger> refCounters = new ConcurrentHashMap<String, AtomicInteger>();
+  private DirectoryReferenceFileGC gc;
+  private IndexInputCloser closer;
+
+  public DirectoryReferenceCounter(Directory directory, DirectoryReferenceFileGC gc, IndexInputCloser closer) {
+    this.directory = directory;
+    this.gc = gc;
+    this.closer = closer;
+  }
+
+  private IndexInput wrap(String name, IndexInput input) {
+    AtomicInteger counter = refCounters.get(name);
+    if (counter == null) {
+      counter = new AtomicInteger();
+      refCounters.put(name, counter);
+    }
+    return new RefIndexInput(input.toString(), input, counter, closer);
+  }
+
+  public void deleteFile(String name) throws IOException {
+    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
+      deleteFile(name);
+      return;
+    }
+    AtomicInteger counter = refCounters.get(name);
+    if (counter != null && counter.get() > 0) {
+      addToFileGC(name);
+    } else {
+      LOG.debug("Delete file [{0}]", name);
+      directory.deleteFile(name);
+    }
+  }
+
+  @Override
+  public IndexOutput createOutput(String name, IOContext context) throws IOException {
+    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
+      return directory.createOutput(name, context);
+    }
+    LOG.debug("Create file [{0}]", name);
+    AtomicInteger counter = refCounters.get(name);
+    if (counter != null) {
+      LOG.error("Unknown error while trying to create ref counter for [{0}] reference exists.", name);
+      throw new IOException("Reference exists [" + name + "]");
+    }
+    refCounters.put(name, new AtomicInteger(0));
+    return directory.createOutput(name, context);
+  }
+
+  @Override
+  public IndexInput openInput(String name, IOContext context) throws IOException {
+    IndexInput input = directory.openInput(name, context);
+    if (name.equals(IndexFileNames.SEGMENTS_GEN)) {
+      return input;
+    }
+    return wrap(name, input);
+  }
+
+  public static class RefIndexInput extends IndexInput {
+
+    private IndexInput input;
+    private AtomicInteger ref;
+    private boolean closed = false;
+    private IndexInputCloser closer;
+
+    public RefIndexInput(String resourceDescription, IndexInput input, AtomicInteger ref, IndexInputCloser closer) {
+      super(resourceDescription);
+      this.input = input;
+      this.ref = ref;
+      this.closer = closer;
+      ref.incrementAndGet();
+      if (closer != null) {
+        closer.add(this);
+      }
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+      // Seems like not all the clones are being closed...
+      if (!closed) {
+        LOG.debug("[{0}] Last resort closing.", input.toString());
+        close();
+      }
+    }
+
+    @Override
+    public RefIndexInput clone() {
+      RefIndexInput refIndexInput = (RefIndexInput) super.clone();
+      if (closer != null) {
+        closer.add(refIndexInput);
+      }
+      refIndexInput.input = (IndexInput) input.clone();
+      refIndexInput.ref.incrementAndGet();
+      return refIndexInput;
+    }
+
+    @Override
+    public void close() throws IOException {
+      if (!closed) {
+        input.close();
+        ref.decrementAndGet();
+        closed = true;
+      }
+    }
+
+    @Override
+    public short readShort() throws IOException {
+      return input.readShort();
+    }
+
+    @Override
+    public int readInt() throws IOException {
+      return input.readInt();
+    }
+
+    @Override
+    public void seek(long pos) throws IOException {
+      input.seek(pos);
+    }
+
+    @Override
+    public int readVInt() throws IOException {
+      return input.readVInt();
+    }
+
+    @Override
+    public String toString() {
+      return input.toString();
+    }
+
+    @Override
+    public long readLong() throws IOException {
+      return input.readLong();
+    }
+
+    @Override
+    public long readVLong() throws IOException {
+      return input.readVLong();
+    }
+
+    @Override
+    public String readString() throws IOException {
+      return input.readString();
+    }
+
+    @Override
+    public long getFilePointer() {
+      return input.getFilePointer();
+    }
+
+    @Override
+    public byte readByte() throws IOException {
+      return input.readByte();
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len) throws IOException {
+      input.readBytes(b, offset, len);
+    }
+
+    @Override
+    public void readBytes(byte[] b, int offset, int len, boolean useBuffer) throws IOException {
+      input.readBytes(b, offset, len, useBuffer);
+    }
+
+    @Override
+    public long length() {
+      return input.length();
+    }
+
+    @Override
+    public Map<String, String> readStringStringMap() throws IOException {
+      return input.readStringStringMap();
+    }
+
+  }
+
+  @Override
+  public void sync(Collection<String> names) throws IOException {
+    directory.sync(names);
+  }
+
+  @Override
+  public void clearLock(String name) throws IOException {
+    directory.clearLock(name);
+  }
+
+  @Override
+  public void close() throws IOException {
+    directory.close();
+  }
+
+  @Override
+  public void setLockFactory(LockFactory lockFactory) throws IOException {
+    directory.setLockFactory(lockFactory);
+  }
+
+  @Override
+  public String getLockID() {
+    return directory.getLockID();
+  }
+
+  @Override
+  public void copy(Directory to, String src, String dest, IOContext context) throws IOException {
+    directory.copy(to, src, dest, context);
+  }
+
+  @Override
+  public boolean fileExists(String name) throws IOException {
+    return directory.fileExists(name);
+  }
+
+  @Override
+  public long fileLength(String name) throws IOException {
+    return directory.fileLength(name);
+  }
+
+  @Override
+  public LockFactory getLockFactory() {
+    return directory.getLockFactory();
+  }
+
+  @Override
+  public String[] listAll() throws IOException {
+    return directory.listAll();
+  }
+
+  @Override
+  public Lock makeLock(String name) {
+    return directory.makeLock(name);
+  }
+
+  @Override
+  public String toString() {
+    return directory.toString();
+  }
+
+  private void addToFileGC(String name) {
+    if (gc != null) {
+      LOG.debug("Add file [{0}] to be GCed once refs are closed.", name);
+      gc.add(directory, name, refCounters);
+    }
+  }
+
+  @Override
+  public Directory getOriginalDirectory() {
+    return directory;
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
new file mode 100644
index 0000000..508ce9d
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/DirectoryReferenceFileGC.java
@@ -0,0 +1,115 @@
+package org.apache.blur.lucene.store.refcounter;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.lucene.store.Directory;
+
+import com.yammer.metrics.Metrics;
+import com.yammer.metrics.core.Gauge;
+import com.yammer.metrics.core.MetricName;
+import static org.apache.blur.metrics.MetricsConstants.*;
+
+public class DirectoryReferenceFileGC extends TimerTask implements Closeable {
+
+
+  private static final Log LOG = LogFactory.getLog(DirectoryReferenceFileGC.class);
+
+  private Timer _timer;
+  private long _delay = 5000;
+  private LinkedBlockingQueue<Value> _queue;
+  private volatile int numberOfFilesToBeDeleted = 0;
+
+  public static class Value {
+    public Value(Directory directory, String name, Map<String, AtomicInteger> refs) {
+      this.directory = directory;
+      this.name = name;
+      this.refs = refs;
+    }
+
+    Directory directory;
+    String name;
+    Map<String, AtomicInteger> refs;
+
+    public boolean tryToDelete() throws IOException {
+      AtomicInteger counter = refs.get(name);
+      if (counter.get() <= 0) {
+        refs.remove(name);
+        LOG.debug("Removing file [{0}]", name);
+        directory.deleteFile(name);
+        return true;
+      } else {
+        LOG.debug("File [{0}] had too many refs [{1}]", name, counter.get());
+      }
+      return false;
+    }
+  }
+
+  public void init() {
+    _timer = new Timer("Blur-File-GC", true);
+    _timer.scheduleAtFixedRate(this, _delay, _delay);
+    _queue = new LinkedBlockingQueue<Value>();
+    Metrics.newGauge(new MetricName(ORG_APACHE_BLUR, LUCENE, FILES_IN_QUEUE_TO_BE_DELETED), new Gauge<Integer>() {
+      @Override
+      public Integer value() {
+        return numberOfFilesToBeDeleted;
+      }
+    });
+  }
+
+  public void add(Directory directory, String name, Map<String, AtomicInteger> refs) {
+    try {
+      _queue.put(new Value(directory, name, refs));
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public void close() {
+    _timer.purge();
+    _timer.cancel();
+  }
+
+  @Override
+  public void run() {
+    Iterator<Value> iterator = _queue.iterator();
+    int count = 0;
+    while (iterator.hasNext()) {
+      Value value = iterator.next();
+      try {
+        if (value.tryToDelete()) {
+          iterator.remove();
+        } else {
+          count++;
+        }
+      } catch (IOException e) {
+        LOG.error("Unknown error", e);
+      }
+    }
+    numberOfFilesToBeDeleted = count;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b0e26648/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/IndexInputCloser.java
----------------------------------------------------------------------
diff --git a/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/IndexInputCloser.java b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/IndexInputCloser.java
new file mode 100644
index 0000000..8a9ee86
--- /dev/null
+++ b/blur-store/src/main/java/org/apache/blur/lucene/store/refcounter/IndexInputCloser.java
@@ -0,0 +1,80 @@
+package org.apache.blur.lucene.store.refcounter;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.ref.ReferenceQueue;
+import java.lang.ref.WeakReference;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.blur.log.Log;
+import org.apache.blur.log.LogFactory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.IOUtils;
+
+public class IndexInputCloser implements Runnable, Closeable {
+
+  private static final Log LOG = LogFactory.getLog(IndexInputCloser.class);
+
+  private ReferenceQueue<IndexInput> referenceQueue = new ReferenceQueue<IndexInput>();
+  private Thread daemon;
+  private AtomicBoolean running = new AtomicBoolean();
+  private Collection<IndexInputCloserRef> refs = new HashSet<IndexInputCloserRef>();
+
+  static class IndexInputCloserRef extends WeakReference<IndexInput> implements Closeable {
+    public IndexInputCloserRef(IndexInput referent, ReferenceQueue<? super IndexInput> q) {
+      super(referent, q);
+    }
+
+    @Override
+    public void close() throws IOException {
+      IndexInput input = get();
+      if (input != null) {
+        LOG.debug("Closing indexinput [{0}]", input);
+        input.close();
+      }
+    }
+  }
+
+  public void init() {
+    running.set(true);
+    daemon = new Thread(this);
+    daemon.setDaemon(true);
+    daemon.setName("IndexIndexCloser");
+    daemon.start();
+  }
+
+  public void add(IndexInput indexInput) {
+    LOG.debug("Adding [{0}]", indexInput);
+    IndexInputCloserRef ref = new IndexInputCloserRef(indexInput, referenceQueue);
+    synchronized (refs) {
+      refs.add(ref);
+    }
+  }
+
+  public void close() {
+    running.set(false);
+    refs.clear();
+    daemon.interrupt();
+  }
+
+  @Override
+  public void run() {
+    while (running.get()) {
+      try {
+        IndexInputCloserRef ref = (IndexInputCloserRef) referenceQueue.remove();
+        LOG.debug("Closing [{0}]", ref);
+        IOUtils.closeWhileHandlingException(ref);
+        synchronized (refs) {
+          refs.remove(ref);
+        }
+      } catch (InterruptedException e) {
+        LOG.debug("Interrupted");
+        running.set(false);
+        return;
+      }
+    }
+  }
+
+}


Mime
View raw message