incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [4/4] git commit: Fixing the argument descriptions, returntype and shell.
Date Wed, 01 Oct 2014 13:56:36 GMT
Fixing the argument descriptions, returntype and shell.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/546d2de8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/546d2de8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/546d2de8

Branch: refs/heads/master
Commit: 546d2de83abb27e31bec7ead84bd6fbaf8d65d38
Parents: fb44fa5
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Wed Oct 1 09:56:29 2014 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Wed Oct 1 09:56:29 2014 -0400

----------------------------------------------------------------------
 .../apache/blur/command/BaseCommandManager.java |  47 +----
 .../java/org/apache/blur/command/Command.java   |  67 ++++++-
 .../commandtype/ClusterExecuteCommand.java      |  15 +-
 .../ClusterExecuteServerReadCommand.java        |  12 ++
 .../commandtype/ClusterIndexReadCommand.java    |  12 ++
 .../commandtype/ClusterServerReadCommand.java   |  16 +-
 .../command/commandtype/IndexReadCommand.java   |  12 ++
 .../command/commandtype/ServerReadCommand.java  |  15 +-
 .../java/org/apache/blur/thrift/TableAdmin.java |  15 +-
 .../blur/command/ShardCommandManagerTest.java   |  14 +-
 .../mapreduce/lib/GenericBlurRecordWriter.java  |   2 +-
 .../org/apache/blur/mapreduce/lib/IOUtil.java   |   3 +
 .../mapreduce/lib/v2/DirectIndexingDriver.java  | 174 ++++++++++++++++++-
 .../shell/DescribePlatformCommandCommand.java   |  23 ++-
 14 files changed, 360 insertions(+), 67 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
index 4779013..e41a32b 100644
--- a/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
+++ b/blur-core/src/main/java/org/apache/blur/command/BaseCommandManager.java
@@ -5,7 +5,6 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.lang.reflect.Method;
 import java.math.BigInteger;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -30,9 +29,6 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.blur.command.annotation.Description;
-import org.apache.blur.command.commandtype.ClusterExecuteServerReadCommand;
-import org.apache.blur.command.commandtype.ServerReadCommand;
-import org.apache.blur.command.commandtype.IndexReadCommand;
 import org.apache.blur.concurrent.Executors;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
@@ -108,12 +104,14 @@ public abstract class BaseCommandManager implements Closeable {
     return new HashMap<String, BigInteger>(_commandLoadTime);
   }
 
-  public Map<String, String> getRequiredArguments(String commandName) {
-    return getArguments(commandName, false);
+  public Set<Argument> getRequiredArguments(String commandName) {
+    Command<?> command = getCommandObject(commandName, null);
+    return command.getRequiredArguments();
   }
 
-  public Map<String, String> getOptionalArguments(String commandName) {
-    return getArguments(commandName, true);
+  public Set<Argument> getOptionalArguments(String commandName) {
+    Command<?> command = getCommandObject(commandName, null);
+    return command.getOptionalArguments();
   }
 
   protected Map<String, String> getArguments(String commandName, boolean optional)
{
@@ -481,38 +479,7 @@ public abstract class BaseCommandManager implements Closeable {
     if (command == null) {
       return null;
     }
-
-    String shardServerReturn;
-    try {
-      if (command instanceof IndexReadCommand) {
-        IndexReadCommand<?> indexReadCommand = (IndexReadCommand<?>) command;
-        Method method = indexReadCommand.getClass().getMethod("execute", new Class[] { IndexContext.class
});
-        Class<?> returnType = method.getReturnType();
-        shardServerReturn = "shard->(" + returnType.getSimpleName() + ")";
-      } else if (command instanceof ServerReadCommand) {
-        ServerReadCommand<?, ?> indexReadCombiningCommand = (ServerReadCommand<?,
?>) command;
-        Method method = indexReadCombiningCommand.getClass().getMethod("combine",
-            new Class[] { CombiningContext.class, Map.class });
-        Class<?> returnType = method.getReturnType();
-        shardServerReturn = "server->(" + returnType.getSimpleName() + ")";
-      } else {
-        shardServerReturn = null;
-      }
-      if (command instanceof ClusterExecuteServerReadCommand) {
-        ClusterExecuteServerReadCommand<?> clusterCommand = (ClusterExecuteServerReadCommand<?>)
command;
-        Method method = clusterCommand.getClass().getMethod("clusterExecute", new Class[]
{ Map.class });
-        Class<?> returnType = method.getReturnType();
-        String clusterReturn = "cluster->(" + returnType.getSimpleName() + ")";
-        if (shardServerReturn == null) {
-          return clusterReturn;
-        } else {
-          return clusterReturn + "," + shardServerReturn;
-        }
-      }
-      return shardServerReturn;
-    } catch (Exception e) {
-      throw new RuntimeException("Unknown error while trying to get return type.", e);
-    }
+    return command.getReturnType();
   }
 
   protected Arguments toArguments(Command<?> command) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/Command.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/Command.java b/blur-core/src/main/java/org/apache/blur/command/Command.java
index f6f7314..89b6e0e 100644
--- a/blur-core/src/main/java/org/apache/blur/command/Command.java
+++ b/blur-core/src/main/java/org/apache/blur/command/Command.java
@@ -17,9 +17,15 @@
 package org.apache.blur.command;
 
 import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Field;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.blur.command.annotation.OptionalArgument;
+import org.apache.blur.command.annotation.RequiredArgument;
 import org.apache.blur.thrift.generated.Blur.Iface;
 
 public abstract class Command<R> implements Cloneable {
@@ -45,11 +51,70 @@ public abstract class Command<R> implements Cloneable {
       throw new RuntimeException(e);
     }
   }
-  
+
   public static <T> Set<T> asSet(T t) {
     Set<T> set = new HashSet<T>();
     set.add(t);
     return set;
   }
 
+  public Set<Argument> getRequiredArguments() {
+    Set<Argument> arguments = new HashSet<Argument>();
+    readArguments(getClass(), arguments, RequiredArgument.class);
+    return arguments;
+  }
+
+  public Set<Argument> getOptionalArguments() {
+    Set<Argument> arguments = new HashSet<Argument>();
+    readArguments(getClass(), arguments, OptionalArgument.class);
+    return arguments;
+  }
+
+  public abstract String getReturnType();
+
+  private void readArguments(Class<?> clazz, Set<Argument> arguments, Class<?
extends Annotation> annotationClass) {
+    if (clazz.equals(Command.class)) {
+      return;
+    }
+    readArguments(clazz.getSuperclass(), arguments, annotationClass);
+    Field[] declaredFields = clazz.getDeclaredFields();
+    for (Field field : declaredFields) {
+      field.setAccessible(true);
+      Annotation annotation = field.getAnnotation(annotationClass);
+      Class<?> type = field.getType();
+      Type genericType = field.getGenericType();
+      Type[] actualTypeArguments = null;
+      if (genericType instanceof ParameterizedType) {
+        ParameterizedType parameterizedType = (ParameterizedType) genericType;
+        actualTypeArguments = parameterizedType.getActualTypeArguments();
+      }
+      String simpleName = getTypeName(type.getSimpleName(), actualTypeArguments);
+      String name = field.getName();
+      if (annotation instanceof RequiredArgument) {
+        RequiredArgument requiredArgument = (RequiredArgument) annotation;
+        String value = requiredArgument.value();
+        arguments.add(new Argument(name, simpleName, value));
+      } else if (annotation instanceof OptionalArgument) {
+        OptionalArgument optionalArgument = (OptionalArgument) annotation;
+        String value = optionalArgument.value();
+        arguments.add(new Argument(name, simpleName, value));
+      }
+    }
+  }
+
+  private String getTypeName(String simpleName, Type[] actualTypeArguments) {
+    if (actualTypeArguments == null) {
+      return simpleName;
+    }
+    StringBuilder builder = new StringBuilder();
+    builder.append(simpleName).append('<');
+    for (Type type : actualTypeArguments) {
+      if (type instanceof Class) {
+        builder.append(((Class<?>) type).getSimpleName());
+      }
+    }
+    builder.append('>');
+    return builder.toString();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java
b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java
index 5ba0863..4312169 100644
--- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java
+++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteCommand.java
@@ -1,13 +1,14 @@
 package org.apache.blur.command.commandtype;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 
 import org.apache.blur.command.ClusterContext;
 import org.apache.blur.command.Command;
 import org.apache.blur.command.CommandRunner;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
 
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -62,4 +63,16 @@ public abstract class ClusterExecuteCommand<T> extends Command<T>
{
       throw new IOException(e);
     }
   }
+
+  @Override
+  public String getReturnType() {
+    try {
+      Method method = getClass().getMethod("clusterExecute", new Class[] { ClusterContext.class
});
+      Class<?> returnType = method.getReturnType();
+      return returnType.getSimpleName();
+    } catch (Exception e) {
+      throw new RuntimeException("Unknown error while trying to get return type.", e);
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java
b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java
index 58d206b..4a7c986 100644
--- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java
+++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterExecuteServerReadCommand.java
@@ -17,6 +17,7 @@
 package org.apache.blur.command.commandtype;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.util.Map;
 
 import org.apache.blur.command.ClusterContext;
@@ -38,6 +39,17 @@ public abstract class ClusterExecuteServerReadCommand<T> extends
Command<T> impl
       InterruptedException;
 
   public abstract T clusterExecute(ClusterContext context) throws IOException, InterruptedException;
+  
+  @Override
+  public String getReturnType() {
+    try {
+      Method method = getClass().getMethod("clusterExecute", new Class[] { ClusterContext.class
});
+      Class<?> returnType = method.getReturnType();
+      return returnType.getSimpleName();
+    } catch (Exception e) {
+      throw new RuntimeException("Unknown error while trying to get return type.", e);
+    }
+  }
 
   @Override
   public T run() throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java
b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java
index ad526d2..83130e7 100644
--- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java
+++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterIndexReadCommand.java
@@ -17,6 +17,7 @@
 package org.apache.blur.command.commandtype;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 
 import org.apache.blur.command.ClusterContext;
 import org.apache.blur.command.Command;
@@ -34,6 +35,17 @@ public abstract class ClusterIndexReadCommand<T1, T2> extends Command<T2>
implem
   public abstract T2 clusterExecute(ClusterContext context) throws IOException, InterruptedException;
 
   @Override
+  public String getReturnType() {
+    try {
+      Method method = getClass().getMethod("clusterExecute", new Class[] { ClusterContext.class
});
+      Class<?> returnType = method.getReturnType();
+      return returnType.getSimpleName();
+    } catch (Exception e) {
+      throw new RuntimeException("Unknown error while trying to get return type.", e);
+    }
+  }
+
+  @Override
   public T2 run() throws IOException {
     try {
       return CommandRunner.run(this);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java
b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java
index ec582f7..be7c0eb 100644
--- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java
+++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ClusterServerReadCommand.java
@@ -17,17 +17,18 @@
 package org.apache.blur.command.commandtype;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.util.Map;
 
 import org.apache.blur.command.CombiningContext;
 import org.apache.blur.command.Command;
 import org.apache.blur.command.CommandRunner;
 import org.apache.blur.command.IndexContext;
-import org.apache.blur.command.ServerRead;
 import org.apache.blur.command.Location;
+import org.apache.blur.command.ServerRead;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
-import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.Blur.Iface;
+import org.apache.blur.thrift.generated.BlurException;
 
 public abstract class ClusterServerReadCommand<T> extends Command<T> implements
ServerRead<T, T> {
 
@@ -37,6 +38,17 @@ public abstract class ClusterServerReadCommand<T> extends Command<T>
implements
       InterruptedException;
 
   @Override
+  public String getReturnType() {
+    try {
+      Method method = getClass().getMethod("combine", new Class[] { CombiningContext.class,
Map.class });
+      Class<?> returnType = method.getReturnType();
+      return returnType.getSimpleName();
+    } catch (Exception e) {
+      throw new RuntimeException("Unknown error while trying to get return type.", e);
+    }
+  }
+
+  @Override
   public T run() throws IOException {
     try {
       return CommandRunner.run(this);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java
b/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java
index 5d2b75d..c0343ea 100644
--- a/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java
+++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/IndexReadCommand.java
@@ -17,6 +17,7 @@
 package org.apache.blur.command.commandtype;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.util.Map;
 
 import org.apache.blur.command.Command;
@@ -33,6 +34,17 @@ public abstract class IndexReadCommand<T> extends Command<Map<Shard,
T>> impleme
   public abstract T execute(IndexContext context) throws IOException, InterruptedException;
 
   @Override
+  public String getReturnType() {
+    try {
+      Method method = getClass().getMethod("execute", new Class[] { IndexContext.class });
+      Class<?> returnType = method.getReturnType();
+      return "map(Shard," + returnType.getSimpleName() + ")";
+    } catch (Exception e) {
+      throw new RuntimeException("Unknown error while trying to get return type.", e);
+    }
+  }
+
+  @Override
   public Map<Shard, T> run() throws IOException {
     try {
       return CommandRunner.run(this);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java
b/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java
index 7a553e9..0523408 100644
--- a/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java
+++ b/blur-core/src/main/java/org/apache/blur/command/commandtype/ServerReadCommand.java
@@ -17,6 +17,7 @@
 package org.apache.blur.command.commandtype;
 
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.util.Map;
 
 import org.apache.blur.command.CombiningContext;
@@ -30,8 +31,7 @@ import org.apache.blur.thirdparty.thrift_0_9_0.TException;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.Blur.Iface;
 
-public abstract class ServerReadCommand<T1, T2> extends Command<Map<Server, T2>>
implements
-    ServerRead<T1, T2> {
+public abstract class ServerReadCommand<T1, T2> extends Command<Map<Server, T2>>
implements ServerRead<T1, T2> {
 
   public abstract T1 execute(IndexContext context) throws IOException, InterruptedException;
 
@@ -39,6 +39,17 @@ public abstract class ServerReadCommand<T1, T2> extends Command<Map<Server,
T2>>
       InterruptedException;
 
   @Override
+  public String getReturnType() {
+    try {
+      Method method = getClass().getMethod("combine", new Class[] { CombiningContext.class,
Map.class });
+      Class<?> returnType = method.getReturnType();
+      return "map(Server," + returnType.getSimpleName() + ")";
+    } catch (Exception e) {
+      throw new RuntimeException("Unknown error while trying to get return type.", e);
+    }
+  }
+
+  @Override
   public Map<Server, T2> run() throws IOException {
     try {
       return CommandRunner.run(this);

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
----------------------------------------------------------------------
diff --git a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
index f396232..d221774 100644
--- a/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
+++ b/blur-core/src/main/java/org/apache/blur/thrift/TableAdmin.java
@@ -34,12 +34,14 @@ import javax.xml.parsers.FactoryConfigurationError;
 import org.apache.blur.BlurConfiguration;
 import org.apache.blur.analysis.FieldManager;
 import org.apache.blur.analysis.FieldTypeDefinition;
+import org.apache.blur.command.Argument;
 import org.apache.blur.command.BaseCommandManager;
 import org.apache.blur.log.Log;
 import org.apache.blur.log.LogFactory;
 import org.apache.blur.manager.clusterstatus.ClusterStatus;
 import org.apache.blur.server.TableContext;
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.ArgumentDescriptor;
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.ColumnDefinition;
@@ -630,14 +632,23 @@ public abstract class TableAdmin implements Iface {
       commandDescriptor.setCommandName(commandName);
       commandDescriptor.setVersion(e.getValue().toString(Character.MAX_RADIX));
       commandDescriptor.setDescription(commandManager.getDescription(commandName));
-      commandDescriptor.setOptionalArguments(commandManager.getOptionalArguments(commandName));
-      commandDescriptor.setRequiredArguments(commandManager.getRequiredArguments(commandName));
+      commandDescriptor.setOptionalArguments(toThrift(commandManager.getOptionalArguments(commandName)));
+      commandDescriptor.setRequiredArguments(toThrift(commandManager.getRequiredArguments(commandName)));
       commandDescriptor.setReturnType(commandManager.getReturnType(commandName));
       result.add(commandDescriptor);
     }
     return result;
   }
 
+  private Map<String, ArgumentDescriptor> toThrift(Set<Argument> arguments) {
+    Map<String, ArgumentDescriptor> result = new HashMap<String, ArgumentDescriptor>();
+    for (Argument argument : arguments) {
+      String name = argument.getName();
+      result.put(name, new ArgumentDescriptor(name, argument.getType(), argument.getDescription()));
+    }
+    return result;
+  }
+
   private void reloadConfig() throws MalformedURLException, FactoryConfigurationError, BException
{
     String blurHome = System.getenv("BLUR_HOME");
     if (blurHome != null) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
----------------------------------------------------------------------
diff --git a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
index 8ca946e..f8ef2e5 100644
--- a/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
+++ b/blur-core/src/test/java/org/apache/blur/command/ShardCommandManagerTest.java
@@ -28,6 +28,7 @@ import java.math.BigInteger;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.SortedSet;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -97,14 +98,17 @@ public class ShardCommandManagerTest {
 
   @Test
   public void testDocumentation() {
-    Map<String, String> requiredArgs = _manager.getRequiredArguments("wait");
-    assertTrue(requiredArgs.containsKey("table"));
+    Set<Argument> requiredArgs = _manager.getRequiredArguments("wait");
     assertEquals(1, requiredArgs.size());
+    assertTrue(requiredArgs.contains(new Argument("table", "String", "The name of the table.")));
 
-    Map<String, String> optionalArgs = _manager.getOptionalArguments("wait");
-    assertTrue(optionalArgs.containsKey("seconds"));
-    assertTrue(optionalArgs.containsKey("shard"));
+    Set<Argument> optionalArgs = _manager.getOptionalArguments("wait");
+    System.out.println(optionalArgs);
     assertEquals(2, optionalArgs.size());
+    assertTrue(optionalArgs.contains(new Argument("seconds", "int",
+        "The number of seconds to sleep, the default is 30 seconds.")));
+    assertTrue(optionalArgs.contains(new Argument("shards", "Set<String>", "The shard
ids (e.g. shard-0000000).")));
+
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
index 4bd311b..8b90331 100644
--- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
+++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/GenericBlurRecordWriter.java
@@ -263,7 +263,7 @@ public class GenericBlurRecordWriter {
     _localTmpPath = null;
   }
 
-  private Record getRecord(BlurRecord value) {
+  public static Record getRecord(BlurRecord value) {
     Record record = new Record();
     record.setRecordId(value.getRecordId());
     record.setFamily(value.getFamily());

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java
index 46c030f..cf62cbc 100644
--- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java
+++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/IOUtil.java
@@ -20,6 +20,9 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
+import org.apache.blur.thrift.generated.Column;
+import org.apache.blur.thrift.generated.Record;
+
 public class IOUtil {
 
   public static final String UTF_8 = "UTF-8";

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java
----------------------------------------------------------------------
diff --git a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java
b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java
index bd56003..5d17aef 100644
--- a/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java
+++ b/blur-mapred-hadoop1/src/main/java/org/apache/blur/mapreduce/lib/v2/DirectIndexingDriver.java
@@ -17,8 +17,12 @@
 package org.apache.blur.mapreduce.lib.v2;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.apache.blur.analysis.FieldManager;
+import org.apache.blur.mapreduce.lib.BlurRecord;
+import org.apache.blur.mapreduce.lib.GenericBlurRecordWriter;
+import org.apache.blur.server.TableContext;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.tools.DFSAdmin;
@@ -26,32 +30,188 @@ import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Mapper.Context;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
+import org.apache.lucene.util.IOUtils;
 
 public class DirectIndexingDriver implements Tool {
 
-  public static class DirectIndexingMapper extends
-      Mapper<IntWritable, DocumentWritable, LuceneKeyWritable, NullWritable> {
+  public static class DirectIndexingMapper extends Mapper<IntWritable, BlurRecord, LuceneKeyWritable,
NullWritable> {
 
     private FieldManager _fieldManager;
+    private TableContext _tableContext;
+    private Analyzer _analyzer;
+
+    // private fieldState.reset();
 
     @Override
     protected void setup(Context context) throws IOException, InterruptedException {
-
+      _fieldManager = _tableContext.getFieldManager();
+      _analyzer = _fieldManager.getAnalyzerForIndex();
     }
 
     @Override
-    protected void map(IntWritable key, DocumentWritable value, Context context) throws IOException,
-        InterruptedException {
+    protected void map(IntWritable key, BlurRecord record, Context context) throws IOException,
InterruptedException {
       int documentId = key.get();
-
+      String rowId = record.getRowId();
+      List<Field> fields = _fieldManager.getFields(rowId, GenericBlurRecordWriter.getRecord(record));
+      // write(documentId, fields, context);
     }
 
+//    public void processFields(final IndexableField[] fields, final int count) throws IOException
{
+//
+//      fieldState.reset();
+//
+//      final boolean doInvert = consumer.start(fields, count);
+//
+//      for (int i = 0; i < count; i++) {
+//
+//        final IndexableField field = fields[i];
+//        final IndexableFieldType fieldType = field.fieldType();
+//
+//        // TODO FI: this should be "genericized" to querying
+//        // consumer if it wants to see this particular field
+//        // tokenized.
+//        if (fieldType.indexed() && doInvert) {
+//          final boolean analyzed = fieldType.tokenized() && docState.analyzer !=
null;
+//
+//          // if the field omits norms, the boost cannot be indexed.
+//          if (fieldType.omitNorms() && field.boost() != 1.0f) {
+//            throw new UnsupportedOperationException("You cannot set an index-time boost:
norms are omitted for field '"
+//                + field.name() + "'");
+//          }
+//
+//          // only bother checking offsets if something will consume them.
+//          // TODO: after we fix analyzers, also check if termVectorOffsets will
+//          // be indexed.
+//          final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+//          int lastStartOffset = 0;
+//
+//          if (i > 0) {
+//            fieldState.position += analyzed ? docState.analyzer.getPositionIncrementGap(fieldInfo.name)
: 0;
+//          }
+//
+//          final TokenStream stream = field.tokenStream(docState.analyzer);
+//          // reset the TokenStream to the first token
+//          stream.reset();
+//
+//          boolean success2 = false;
+//
+//          try {
+//            boolean hasMoreTokens = stream.incrementToken();
+//
+//            fieldState.attributeSource = stream;
+//
+//            OffsetAttribute offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
+//            PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource
+//                .addAttribute(PositionIncrementAttribute.class);
+//
+//            if (hasMoreTokens) {
+//              consumer.start(field);
+//
+//              do {
+//                // If we hit an exception in stream.next below
+//                // (which is fairly common, eg if analyzer
+//                // chokes on a given document), then it's
+//                // non-aborting and (above) this one document
+//                // will be marked as deleted, but still
+//                // consume a docID
+//
+//                final int posIncr = posIncrAttribute.getPositionIncrement();
+//                if (posIncr < 0) {
+//                  throw new IllegalArgumentException("position increment must be >=0
(got " + posIncr + ")");
+//                }
+//                if (fieldState.position == 0 && posIncr == 0) {
+//                  throw new IllegalArgumentException("first position increment must be
> 0 (got 0)");
+//                }
+//                int position = fieldState.position + posIncr;
+//                if (position > 0) {
+//                  // NOTE: confusing: this "mirrors" the
+//                  // position++ we do below
+//                  position--;
+//                } else if (position < 0) {
+//                  throw new IllegalArgumentException("position overflow for field '" +
field.name() + "'");
+//                }
+//
+//                // position is legal, we can safely place it in fieldState now.
+//                // not sure if anything will use fieldState after non-aborting
+//                // exc...
+//                fieldState.position = position;
+//
+//                if (posIncr == 0)
+//                  fieldState.numOverlap++;
+//
+//                if (checkOffsets) {
+//                  int startOffset = fieldState.offset + offsetAttribute.startOffset();
+//                  int endOffset = fieldState.offset + offsetAttribute.endOffset();
+//                  if (startOffset < 0 || endOffset < startOffset) {
+//                    throw new IllegalArgumentException(
+//                        "startOffset must be non-negative, and endOffset must be >=
startOffset, " + "startOffset="
+//                            + startOffset + ",endOffset=" + endOffset);
+//                  }
+//                  if (startOffset < lastStartOffset) {
+//                    throw new IllegalArgumentException("offsets must not go backwards startOffset="
+ startOffset
+//                        + " is < lastStartOffset=" + lastStartOffset);
+//                  }
+//                  lastStartOffset = startOffset;
+//                }
+//
+//                boolean success = false;
+//                try {
+//                  // If we hit an exception in here, we abort
+//                  // all buffered documents since the last
+//                  // flush, on the likelihood that the
+//                  // internal state of the consumer is now
+//                  // corrupt and should not be flushed to a
+//                  // new segment:
+//                  consumer.add();
+//                  success = true;
+//                } finally {
+//                  if (!success) {
+//                    docState.docWriter.setAborting();
+//                  }
+//                }
+//                fieldState.length++;
+//                fieldState.position++;
+//              } while (stream.incrementToken());
+//            }
+//            // trigger streams to perform end-of-stream operations
+//            stream.end();
+//
+//            fieldState.offset += offsetAttribute.endOffset();
+//            success2 = true;
+//          } finally {
+//            if (!success2) {
+//              IOUtils.closeWhileHandlingException(stream);
+//            } else {
+//              stream.close();
+//            }
+//          }
+//
+//          fieldState.offset += analyzed ? docState.analyzer.getOffsetGap(fieldInfo.name)
: 0;
+//          fieldState.boost *= field.boost();
+//        }
+//
+//        // LUCENE-2387: don't hang onto the field, so GC can
+//        // reclaim
+//        fields[i] = null;
+//      }
+//
+//      consumer.finish();
+//      endConsumer.finish();
+//    }
+
   }
 
   private Configuration _conf;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/546d2de8/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java
----------------------------------------------------------------------
diff --git a/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java
b/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java
index 5ab7c89..1541010 100644
--- a/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java
+++ b/blur-shell/src/main/java/org/apache/blur/shell/DescribePlatformCommandCommand.java
@@ -24,6 +24,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 
 import org.apache.blur.thirdparty.thrift_0_9_0.TException;
+import org.apache.blur.thrift.generated.ArgumentDescriptor;
 import org.apache.blur.thrift.generated.Blur;
 import org.apache.blur.thrift.generated.BlurException;
 import org.apache.blur.thrift.generated.CommandDescriptor;
@@ -64,25 +65,35 @@ public class DescribePlatformCommandCommand extends Command implements
CommandFi
       out.println(addWhiteSpace("Version:", width) + version);
     }
 
-    Map<String, String> requiredArguments = commandDescriptor.getRequiredArguments();
+    Map<String, ArgumentDescriptor> requiredArguments = commandDescriptor.getRequiredArguments();
     if (requiredArguments != null && !requiredArguments.isEmpty()) {
       out.println();
       out.println("Required Arguments:");
-      for (Entry<String, String> e : requiredArguments.entrySet()) {
-        out.println(addWhiteSpace("-" + e.getKey(), width) + e.getValue());
+      for (Entry<String, ArgumentDescriptor> e : requiredArguments.entrySet()) {
+        out.println(addWhiteSpace("-" + e.getKey(), width) + toString(e.getValue()));
       }
     }
-    Map<String, String> optionalArguments = commandDescriptor.getOptionalArguments();
+    Map<String, ArgumentDescriptor> optionalArguments = commandDescriptor.getOptionalArguments();
     if (optionalArguments != null && !optionalArguments.isEmpty()) {
       out.println();
       out.println("Optional Arguments");
-      for (Entry<String, String> e : optionalArguments.entrySet()) {
-        out.println(addWhiteSpace("-" + e.getKey(), width) + e.getValue());
+      for (Entry<String, ArgumentDescriptor> e : optionalArguments.entrySet()) {
+        out.println(addWhiteSpace("-" + e.getKey(), width) + toString(e.getValue()));
       }
     }
     out.println();
   }
 
+  private String toString(ArgumentDescriptor argumentDescriptor) {
+    String type = argumentDescriptor.getType();
+    String description = argumentDescriptor.getDescription();
+    if (description == null) {
+      return "Type: [" + type + "]";
+    } else {
+      return "Type: [" + type + "] Description: " + description;
+    }
+  }
+
   @Override
   public String description() {
     return "Describes the specifed command.";


Mime
View raw message