incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Adding some more tests and adding default documentation on standard commands.
Date Tue, 23 Sep 2014 17:57:13 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 6092e1172 -> d92fbf629


Adding some more tests and adding default documentation on standard commands.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d92fbf62
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d92fbf62
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d92fbf62

Branch: refs/heads/master
Commit: d92fbf62965c8f4d81e15328184b7371f81be3de
Parents: 6092e11
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Sep 23 13:57:00 2014 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Sep 23 13:57:00 2014 -0400

----------------------------------------------------------------------
 .../org/apache/blur/command/DocumentCount.java  |  8 ---
 .../blur/command/DocumentCountCombiner.java     | 12 ----
 .../blur/command/DocumentCountNoCombine.java    | 11 ----
 .../apache/blur/command/BaseCommandManager.java | 58 +++++++++++++++-----
 .../java/org/apache/blur/command/Command.java   |  6 ++
 .../blur/command/ControllerCommandManager.java  |  2 +-
 .../blur/command/ShardCommandManager.java       | 14 ++++-
 .../blur/command/ShardCommandManagerTest.java   | 38 ++++++++++---
 .../org/apache/blur/command/WaitForSeconds.java | 12 +---
 .../services/org.apache.blur.command.Commands   |  3 +-
 10 files changed, 96 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java
----------------------------------------------------------------------
diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java b/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java
index 0eff004..bc104e7 100644
--- a/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java
+++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCount.java
@@ -18,18 +18,10 @@ package org.apache.blur.command;
 
 import java.io.IOException;
 
-import org.apache.blur.command.Command;
-import org.apache.blur.command.IndexContext;
-import org.apache.blur.command.IndexReadCommand;
-import org.apache.blur.command.annotation.Argument;
 import org.apache.blur.command.annotation.Description;
-import org.apache.blur.command.annotation.RequiredArguments;
-import org.apache.blur.command.annotation.OptionalArguments;
 
 @SuppressWarnings("serial")
 @Description("Gets the number of visible documents in the index.")
-@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute
the document count command.", type = String.class) })
-@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document
count command.", type = String.class) })
 public class DocumentCount extends Command implements IndexReadCommand<Integer> {
 
   private static final String DOC_COUNT = "docCount";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java
----------------------------------------------------------------------
diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java
b/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java
index 8d0abc2..cf64ed8 100644
--- a/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java
+++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCountCombiner.java
@@ -20,22 +20,10 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.blur.command.Command;
-import org.apache.blur.command.ClusterCommand;
-import org.apache.blur.command.ClusterContext;
-import org.apache.blur.command.IndexContext;
-import org.apache.blur.command.IndexReadCombiningCommand;
-import org.apache.blur.command.Server;
-import org.apache.blur.command.Shard;
-import org.apache.blur.command.annotation.Argument;
 import org.apache.blur.command.annotation.Description;
-import org.apache.blur.command.annotation.RequiredArguments;
-import org.apache.blur.command.annotation.OptionalArguments;
 
 @SuppressWarnings("serial")
 @Description("Gets the number of visible documents in the index.")
-@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute
the document count command.", type = String.class) })
-@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document
count command.", type = String.class) })
 public class DocumentCountCombiner extends Command implements ClusterCommand<Long>,
     IndexReadCombiningCommand<Integer, Long> {
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java
----------------------------------------------------------------------
diff --git a/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java
b/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java
index e7a74c6..3103ff2 100644
--- a/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java
+++ b/blur-command/src/main/java/org/apache/blur/command/DocumentCountNoCombine.java
@@ -20,21 +20,10 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.blur.command.Command;
-import org.apache.blur.command.ClusterCommand;
-import org.apache.blur.command.ClusterContext;
-import org.apache.blur.command.IndexContext;
-import org.apache.blur.command.IndexReadCommand;
-import org.apache.blur.command.Shard;
-import org.apache.blur.command.annotation.Argument;
 import org.apache.blur.command.annotation.Description;
-import org.apache.blur.command.annotation.RequiredArguments;
-import org.apache.blur.command.annotation.OptionalArguments;
 
 @SuppressWarnings("serial")
 @Description("Gets the number of visible documents in the index.")
-@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute
the document count command.", type = String.class) })
-@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document
count command.", type = String.class) })
 public class DocumentCountNoCombine extends Command implements IndexReadCommand<Integer>,
ClusterCommand<Long> {
 
   private static final String DOC_COUNT_NO_COMBINE = "docCountNoCombine";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
index b9c713c..e246b8a 100644
--- a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
+++ b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
@@ -126,16 +126,30 @@ public class BaseCommandManager implements Closeable {
     Class<? extends Command> clazz = command.getClass();
     Map<String, String> arguments = new TreeMap<String, String>();
     Argument[] args = getArgumentArray(clazz, optional);
-    if (args == null) {
-      return arguments;
-    }
-    for (Argument argument : args) {
-      Class<?> type = argument.type();
-      arguments.put(argument.name(), ("(" + type.getSimpleName() + ") " + argument.value()).trim());
+    addArguments(arguments, args);
+    if (optional) {
+      if (!(command instanceof ShardRoute)) {
+        Argument[] argumentArray = getArgumentArray(Command.class, optional);
+        addArguments(arguments, argumentArray);
+      }
+    } else {
+      if (!(command instanceof TableRoute)) {
+        Argument[] argumentArray = getArgumentArray(Command.class, optional);
+        addArguments(arguments, argumentArray);
+      }
     }
     return arguments;
   }
 
+  private void addArguments(Map<String, String> arguments, Argument[] args) {
+    if (args != null) {
+      for (Argument argument : args) {
+        Class<?> type = argument.type();
+        arguments.put(argument.name(), ("(" + type.getSimpleName() + ") " + argument.value()).trim());
+      }
+    }
+  }
+
   protected Argument[] getArgumentArray(Class<? extends Command> clazz, boolean optional)
{
     if (optional) {
       OptionalArguments arguments = clazz.getAnnotation(OptionalArguments.class);
@@ -165,23 +179,26 @@ public class BaseCommandManager implements Closeable {
     };
   }
 
-  public void commandRefresh() throws IOException {
-    loadNewCommandsFromCommandPath();
+  public int commandRefresh() throws IOException {
+    return loadNewCommandsFromCommandPath();
   }
 
-  protected synchronized void loadNewCommandsFromCommandPath() throws IOException {
+  protected synchronized int loadNewCommandsFromCommandPath() throws IOException {
     Path path = new Path(_commandPath);
     FileSystem fileSystem = path.getFileSystem(_configuration);
     FileStatus[] listStatus = fileSystem.listStatus(path);
+    int changeCount = 0;
     for (FileStatus fileStatus : listStatus) {
       BigInteger contentsCheck = checkContents(fileStatus, fileSystem);
       Path entryPath = fileStatus.getPath();
       BigInteger currentValue = _commandPathLastChange.get(entryPath);
       if (!contentsCheck.equals(currentValue)) {
+        changeCount++;
         loadNewCommand(fileSystem, fileStatus, contentsCheck);
         _commandPathLastChange.put(entryPath, contentsCheck);
       }
     }
+    return changeCount;
   }
 
   protected void loadNewCommand(FileSystem fileSystem, FileStatus fileStatus, BigInteger
hashOfContents)
@@ -237,6 +254,7 @@ public class BaseCommandManager implements Closeable {
 
   protected BigInteger checkContents(FileStatus fileStatus, FileSystem fileSystem) throws
IOException {
     if (fileStatus.isDir()) {
+      LOG.info("Scanning directory [{0}].", fileStatus.getPath());
       BigInteger count = BigInteger.ZERO;
       Path path = fileStatus.getPath();
       FileStatus[] listStatus = fileSystem.listStatus(path);
@@ -245,7 +263,15 @@ public class BaseCommandManager implements Closeable {
       }
       return count;
     } else {
-      return BigInteger.valueOf(fileStatus.getModificationTime());
+      int hashCode = fileStatus.getPath().toString().hashCode();
+      long modificationTime = fileStatus.getModificationTime();
+      long len = fileStatus.getLen();
+      BigInteger bi = BigInteger.valueOf(hashCode).add(
+          BigInteger.valueOf(modificationTime).add(BigInteger.valueOf(len)));
+      LOG.info("File path hashcode [{0}], mod time [{1}], len [{2}] equals file code [{3}].",
+          Integer.toString(hashCode), Long.toString(modificationTime), Long.toString(len),
+          bi.toString(Character.MAX_RADIX));
+      return bi;
     }
   }
 
@@ -296,7 +322,8 @@ public class BaseCommandManager implements Closeable {
     }
   }
 
-  protected Response submitDriverCallable(Callable<Response> callable) throws IOException,
TimeoutException {
+  protected Response submitDriverCallable(Callable<Response> callable) throws IOException,
TimeoutException,
+      ExceptionCollector {
     ExecutionContext executionContext = ExecutionContext.create();
     Future<Response> future = _executorServiceDriver.submit(executionContext.wrapCallable(callable));
     executionContext.registerDriverFuture(future);
@@ -309,7 +336,11 @@ public class BaseCommandManager implements Closeable {
     } catch (InterruptedException e) {
       throw new IOException(e);
     } catch (ExecutionException e) {
-      throw new IOException(e.getCause());
+      Throwable cause = e.getCause();
+      if (cause instanceof ExceptionCollector) {
+        throw (ExceptionCollector) cause;
+      }
+      throw new IOException(cause);
     } catch (java.util.concurrent.TimeoutException e) {
       LOG.info("Timeout of command [{0}]", executionId);
       throw new TimeoutException(executionId);
@@ -355,7 +386,8 @@ public class BaseCommandManager implements Closeable {
     return _commandNameLookup.get(clazz);
   }
 
-  protected Map<String, Set<Shard>> getShards(TableContextFactory tableContextFactory,
Command command, final Args args, Set<String> tables) throws IOException {
+  protected Map<String, Set<Shard>> getShards(TableContextFactory tableContextFactory,
Command command,
+      final Args args, Set<String> tables) throws IOException {
     Map<String, Set<Shard>> shardMap = new TreeMap<String, Set<Shard>>();
     if (command instanceof ShardRoute) {
       ShardRoute shardRoute = (ShardRoute) command;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/Command.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/Command.java b/blur-core/src/main/java/org/apache/blur/command/Command.java
index a063629..fadcd28 100644
--- a/blur-core/src/main/java/org/apache/blur/command/Command.java
+++ b/blur-core/src/main/java/org/apache/blur/command/Command.java
@@ -18,7 +18,13 @@ package org.apache.blur.command;
 
 import java.io.Serializable;
 
+import org.apache.blur.command.annotation.Argument;
+import org.apache.blur.command.annotation.OptionalArguments;
+import org.apache.blur.command.annotation.RequiredArguments;
+
 @SuppressWarnings("serial")
+@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute
the document count command.", type = String.class) })
+@OptionalArguments({ @Argument(name = "shard", value = "The shard id to execute the document
count command.", type = String.class) })
 public abstract class Command implements Serializable, Cloneable {
 
   public abstract String getName();

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java
b/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java
index e33b8be..b30e23d 100644
--- a/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java
+++ b/blur-core/src/main/java/org/apache/blur/command/ControllerCommandManager.java
@@ -33,7 +33,7 @@ public class ControllerCommandManager extends BaseCommandManager {
   }
 
   public Response execute(TableContextFactory tableContextFactory, LayoutFactory layoutFactory,
String commandName,
-      final Args args) throws IOException, TimeoutException {
+      final Args args) throws IOException, TimeoutException, ExceptionCollector {
     final ClusterContext context = createCommandContext(tableContextFactory, layoutFactory,
args);
     final Command command = getCommandObject(commandName);
     if (command == null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java
index 92909fe..b9565b5 100644
--- a/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java
+++ b/blur-core/src/main/java/org/apache/blur/command/ShardCommandManager.java
@@ -47,7 +47,7 @@ public class ShardCommandManager extends BaseCommandManager {
   }
 
   public Response execute(final TableContextFactory tableContextFactory, final String commandName,
final Args args)
-      throws IOException, TimeoutException {
+      throws IOException, TimeoutException, ExceptionCollector {
     final ShardServerContext shardServerContext = getShardServerContext();
     Callable<Response> callable = new Callable<Response>() {
       @Override
@@ -106,7 +106,7 @@ public class ShardCommandManager extends BaseCommandManager {
   }
 
   private Map<Shard, Object> executeReadCommand(ShardServerContext shardServerContext,
Command command,
-      final TableContextFactory tableContextFactory, final Args args) throws IOException
{
+      final TableContextFactory tableContextFactory, final Args args) throws IOException,
ExceptionCollector {
     Set<String> tables = getTables(command, args);
     if (tables.isEmpty()) {
       throw new IOException("At least one table needs to specified.");
@@ -145,6 +145,7 @@ public class ShardCommandManager extends BaseCommandManager {
       }
     }
     Map<Shard, Object> resultMap = new HashMap<Shard, Object>();
+    ExceptionCollector collector = null;
     for (Entry<Shard, Future<?>> e : futureMap.entrySet()) {
       Shard shard = e.getKey();
       Future<?> future = e.getValue();
@@ -154,10 +155,17 @@ public class ShardCommandManager extends BaseCommandManager {
       } catch (InterruptedException ex) {
         throw new IOException(ex);
       } catch (ExecutionException ex) {
-        throw new IOException(ex.getCause());
+        if (collector == null) {
+          collector = new ExceptionCollector();
+        }
+        collector.add(ex.getCause());
+        continue;
       }
       resultMap.put(shard, object);
     }
+    if (collector != null) {
+      throw collector;
+    }
     return resultMap;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
index 2e992d9..6047bfa 100644
--- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
@@ -18,6 +18,7 @@ package org.apache.blur.command;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -87,9 +88,10 @@ public class ShardCommandManagerTest {
   @Test
   public void testGetCommands() {
     Map<String, BigInteger> commands = _manager.getCommands();
-    assertEquals(1, commands.size());
+    assertEquals(2, commands.size());
     assertTrue(commands.containsKey("wait"));
-    assertEquals(BigInteger.ZERO,  commands.get("wait"));
+    assertTrue(commands.containsKey("error"));
+    assertEquals(BigInteger.ZERO, commands.get("wait"));
   }
 
   @Test
@@ -105,15 +107,18 @@ public class ShardCommandManagerTest {
   }
 
   @Test
-  public void testNewCommandLoading() throws IOException, TimeoutException, InterruptedException
{
+  public void testNewCommandLoading() throws IOException, TimeoutException, InterruptedException,
ExceptionCollector {
     _manager.close();
     new File(_tmpPath).mkdirs();
     File commandPath = new File(_commandPath);
     rmr(commandPath);
+    if (commandPath.exists()) {
+      fail("Command path [" + commandPath + "] still exists.");
+    }
     commandPath.mkdirs();
     {
       InputStream inputStream = getClass().getResourceAsStream("/org/apache/blur/command/test1/test1.jar");
-      File dest = new File(commandPath, "test.jar");
+      File dest = new File(commandPath, "test1.jar");
       FileOutputStream output = new FileOutputStream(dest);
       IOUtils.copy(inputStream, output);
       inputStream.close();
@@ -133,13 +138,14 @@ public class ShardCommandManagerTest {
 
     {
       InputStream inputStream = getClass().getResourceAsStream("/org/apache/blur/command/test2/test2.jar");
-      File dest = new File(commandPath, "test.jar");
+      File dest = new File(commandPath, "test2.jar");
       FileOutputStream output = new FileOutputStream(dest);
       IOUtils.copy(inputStream, output);
       inputStream.close();
       output.close();
     }
-    manager.commandRefresh();
+
+    assertEquals(1, manager.commandRefresh());
 
     {
       Args args = new Args();
@@ -168,7 +174,7 @@ public class ShardCommandManagerTest {
   }
 
   @Test
-  public void testShardCommandManagerNormalWait() throws IOException, TimeoutException {
+  public void testShardCommandManagerNormalWait() throws IOException, TimeoutException, ExceptionCollector
{
     Response response;
     ExecutionId executionId = null;
 
@@ -193,7 +199,23 @@ public class ShardCommandManagerTest {
   }
 
   @Test
-  public void testShardCommandManagerNormalWithCancel() throws IOException, TimeoutException
{
+  public void testShardCommandManagerErrorWait() throws IOException, TimeoutException, ExceptionCollector
{
+    Args args = new Args();
+    args.set("seconds", 1);
+    args.set("table", "test");
+    TableContextFactory tableContextFactory = getTableContextFactory();
+    try {
+      _manager.execute(tableContextFactory, "error", args);
+      fail();
+    } catch (ExceptionCollector e) {
+      Throwable t = e.getCause();
+      assertTrue(t instanceof RuntimeException);
+      assertEquals("error-test", t.getMessage());
+    }
+  }
+
+  @Test
+  public void testShardCommandManagerNormalWithCancel() throws IOException, TimeoutException,
ExceptionCollector {
     Response response;
     ExecutionId executionId = null;
 

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java b/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java
index 32ce8a0..15476f5 100644
--- a/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java
+++ b/blur-core/src/test/java/org/apache/blur/command/WaitForSeconds.java
@@ -19,21 +19,11 @@ package org.apache.blur.command;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.blur.command.Command;
-import org.apache.blur.command.IndexContext;
-import org.apache.blur.command.IndexReadCommand;
 import org.apache.blur.command.annotation.Argument;
-import org.apache.blur.command.annotation.RequiredArguments;
 import org.apache.blur.command.annotation.OptionalArguments;
 
 @SuppressWarnings("serial")
-@RequiredArguments({ @Argument(name = "table", value = "The name of the table to execute
the wait for N number of seconds command.", type = String.class) })
-@OptionalArguments({
-
-    @Argument(name = "shard", value = "The shard id to execute the wait for N number of seconds
command.", type = String.class),
-    @Argument(name = "seconds", value = "The number of seconds to sleep, the default is 30
seconds.", type = Integer.class)
-
-})
+@OptionalArguments({ @Argument(name = "seconds", value = "The number of seconds to sleep,
the default is 30 seconds.", type = Integer.class) })
 public class WaitForSeconds extends Command implements IndexReadCommand<Boolean> {
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d92fbf62/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands
----------------------------------------------------------------------
diff --git a/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands
b/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands
index 51c8777..f06a262 100644
--- a/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands
+++ b/blur-core/src/test/resources/META-INF/services/org.apache.blur.command.Commands
@@ -13,4 +13,5 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-org.apache.blur.command.WaitForSeconds
\ No newline at end of file
+org.apache.blur.command.WaitForSeconds
+org.apache.blur.command.ThrowException
\ No newline at end of file


Mime
View raw message