incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [1/29] git commit: More fixes based issues found during command creation. The biggest issue is that Lucene field infos are not consistent between indexing and fetching.
Date Fri, 04 Jan 2013 22:42:52 GMT
More fixes based issues found during command creation.  The biggest issue is that Lucene field
infos are not consistent between indexing and fetching.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/d709ddb8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/d709ddb8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/d709ddb8

Branch: refs/heads/0.2-dev
Commit: d709ddb803a9bdd46abbd4a04286fa3b592ba382
Parents: eb5421c
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Fri Jan 4 16:24:32 2013 -0500
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Fri Jan 4 16:24:32 2013 -0500

----------------------------------------------------------------------
 .../java/org/apache/blur/thrift/BlurServer.java    |    5 +-
 .../apache/blur/utils/ThriftLuceneConversion.java  |   11 +-
 .../java/org/apache/blur/shell/AddCommand.java     |  157 +++++++++++----
 .../java/org/apache/blur/shell/CreateCommand.java  |    9 +-
 .../org/apache/blur/shell/DescribeCommand.java     |    2 +-
 .../org/apache/blur/shell/DisableTableCommand.java |    2 +-
 .../org/apache/blur/shell/EnableTableCommand.java  |    2 +-
 7 files changed, 139 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
index 3c5ae36..c148d3b 100644
--- a/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
+++ b/src/blur-core/src/main/java/org/apache/blur/thrift/BlurServer.java
@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -69,6 +70,7 @@ import org.apache.blur.utils.BlurConstants;
 import org.apache.blur.utils.BlurUtil;
 import org.apache.blur.utils.ThriftLuceneConversion;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.thrift.TException;
 
@@ -326,7 +328,8 @@ public class BlurServer extends TableAdmin implements Iface {
         if (searcher == null) {
           result.addAll(forwardDoc(session, shardIndex, docLocation, fieldsToLoad));
         } else {
-          result.add(toThrift(searcher.document(docId, fieldsToLoad)));
+          org.apache.lucene.document.Document document = searcher.document(docId, fieldsToLoad);
+          result.add(toThrift(document));
         }
       }
       return result;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
----------------------------------------------------------------------
diff --git a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
index a71210e..12163ce 100644
--- a/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
+++ b/src/blur-core/src/main/java/org/apache/blur/utils/ThriftLuceneConversion.java
@@ -32,7 +32,6 @@ import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
-import org.apache.lucene.queryparser.classic.QueryParserBase;
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
@@ -115,8 +114,14 @@ public class ThriftLuceneConversion {
         result.setType(TYPE.BINARY);
         result.setValue(toByteBuffer(bytes));
       } else {
-        result.setType(TYPE.STRING);
-        result.setValue(toByteBuffer(field.stringValue()));
+        //@TODO this is a bit of a hack
+        if (field.fieldType().omitNorms()) {
+          result.setType(TYPE.STRING);
+          result.setValue(toByteBuffer(field.stringValue()));  
+        } else{
+          result.setType(TYPE.TEXT);
+          result.setValue(toByteBuffer(field.stringValue()));
+        }
       }
     }
     return result;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java
----------------------------------------------------------------------
diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java
index 71c989a..e960eb7 100644
--- a/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java
+++ b/src/blur-shell/src/main/java/org/apache/blur/shell/AddCommand.java
@@ -19,59 +19,146 @@
 package org.apache.blur.shell;
 
 import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
 
 import jline.console.ConsoleReader;
 
 import org.apache.blur.thrift.generated.Blur.Iface;
 import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.Document;
+import org.apache.blur.thrift.generated.Field;
+import org.apache.blur.thrift.generated.MutateOptions;
+import org.apache.blur.thrift.generated.TYPE;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
 import org.apache.thrift.TException;
 
 public class AddCommand extends Command {
+  private static final String HELP = "help";
+  private static final String WAL = "wal";
+  private static final String VISIBLE = "visible";
+  private static final String SHARD = "shard";
+
   public AddCommand(Iface client) {
     super(client);
   }
 
+  @SuppressWarnings("unchecked")
   @Override
-  public void doit(PrintWriter out, String[] args, ConsoleReader reader) throws CommandException,
TException, BlurException {
-    if (args.length != 7) {
-      throw new CommandException("Invalid args: " + help());
+  public void doit(PrintWriter out, String[] args, ConsoleReader reader) throws CommandException,
TException, BlurException, ParseException {
+    Options options = new Options();
+    options.addOption("h", HELP, false, "Print the help for this command.");
+    options.addOption("s", SHARD, true, "Shard to add the documents.");
+    options.addOption("v", VISIBLE, false, "Wait for the data to be visible.");
+    options.addOption("w", WAL, false, "Enable write ahead log.");
+
+    CommandLineParser parser = new PosixParser();
+    CommandLine line = parser.parse(options, args);
+    List<String> argList = line.getArgList();
+
+    if (line.hasOption(HELP) || argList.size() < 3) {
+      HelpFormatter formatter = new HelpFormatter();
+      formatter.printHelp(help(), options);
+      return;
+    }
+
+    String tablename = argList.get(1);
+
+    Iface client = getClient();
+    MutateOptions moptions = new MutateOptions();
+    moptions.setTable(tablename);
+    if (line.hasOption(SHARD)) {
+      moptions.setShardIndex(Integer.parseInt(line.getOptionValue(SHARD)));
+    }
+    if (line.hasOption(VISIBLE)) {
+      moptions.setWaitToBeVisible(true);
+    } else {
+      moptions.setWaitToBeVisible(false);
+    }
+
+    if (line.hasOption(WAL)) {
+      moptions.setWriteAheadLog(true);
+    } else {
+      moptions.setWriteAheadLog(false);
+    }
+
+    Document doc = new Document();
+
+    for (int i = 2; i < argList.size(); i++) {
+      Field field = getField(argList.get(i));
+      doc.addToFields(field);
+    }
+
+    List<Document> documents = new ArrayList<Document>();
+    documents.add(doc);
+    client.addDocuments(moptions, documents);
+  }
+
+  private Field getField(String arg) throws CommandException {
+    Field field = new Field();
+    String name;
+    TYPE type = TYPE.TEXT;
+    float boost = 1.0f;
+    int indexOfEquals = arg.indexOf(':');
+    if (indexOfEquals < 0) {
+      throw new CommandException("No value specified [" + arg + "]");
+    }
+    int startOfType = arg.indexOf('/');
+    if (startOfType < 0) {
+      // no type or boost
+      name = arg.substring(0, indexOfEquals);
+    } else {
+      name = arg.substring(0, startOfType);
+
+      int startOfBoost = arg.indexOf('/', startOfType + 1);
+      if (startOfBoost < 0) {
+        // no boost
+        type = TYPE.valueOf(arg.substring(startOfType + 1, indexOfEquals));
+      } else {
+        type = TYPE.valueOf(arg.substring(startOfType + 1, startOfBoost));
+        boost = Float.parseFloat(arg.substring(startOfBoost + 1, indexOfEquals));
+      }
+    }
+    field.setName(name);
+    field.setBoost(boost);
+    field.setType(type);
+    field.setValue(getValue(type, arg.substring(indexOfEquals + 1)));
+    return field;
+  }
+
+  private byte[] getValue(TYPE type, String valueString) throws CommandException {
+    switch (type) {
+    case TEXT:
+    case STRING:
+    case BINARY:
+      try {
+        return valueString.getBytes("UTF-8");
+      } catch (UnsupportedEncodingException e) {
+        throw new RuntimeException(e);
+      }
+    case DOUBLE:
+      return ByteBuffer.allocate(8).putDouble(Double.parseDouble(valueString)).array();
+    case LONG:
+      return ByteBuffer.allocate(8).putLong(Long.parseLong(valueString)).array();
+    case INT:
+      return ByteBuffer.allocate(4).putInt(Integer.parseInt(valueString)).array();
+    case FLOAT:
+      return ByteBuffer.allocate(4).putFloat(Float.parseFloat(valueString)).array();
+    default:
+      throw new CommandException("Unknown type [" + type + "]");
     }
-    String tablename = args[1];
-    String rowid = args[2];
-    String recordid = args[3];
-    String columnfamily = args[4];
-    String columnname = args[5];
-    String value = args[6];
-
-    throw new CommandException("Needs to be implemented");
-
-    // List<Column> columns = new ArrayList<Column>();
-    // columns.add(new Column(columnname, value));
-    //
-    // Record record = new Record();
-    // record.setRecordId(recordid);
-    // record.setFamily(columnfamily);
-    // record.setColumns(columns);
-    //
-    // RecordMutation recordMutation = new RecordMutation();
-    // recordMutation.setRecord(record);
-    // recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD);
-    //
-    // List<RecordMutation> recordMutations = new ArrayList<RecordMutation>();
-    // recordMutations.add(recordMutation);
-    //
-    // RowMutation mutation = new RowMutation();
-    // mutation.setTable(tablename);
-    // mutation.setRowId(rowid);
-    // mutation.setRowMutationType(RowMutationType.REPLACE_ROW);
-    // mutation.setRecordMutations(recordMutations);
-    //
-    // client.mutate(mutation);
   }
 
   @Override
   public String help() {
-    return "mutate the specified row, args; tablename rowid recordid columnfamily columnname
value";
+    return "add <tablename> <field[/type[/boost]]:value> ...";
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java
----------------------------------------------------------------------
diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java
index 8c49ae4..4a283b6 100644
--- a/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java
+++ b/src/blur-shell/src/main/java/org/apache/blur/shell/CreateCommand.java
@@ -59,14 +59,9 @@ public class CreateCommand extends Command {
     try {
       CommandLine line = parser.parse(options, args);
       List<String> argList = line.getArgList();
-      boolean printHelp = false;
       if (line.hasOption(HELP) || argList.size() != 4) {
-        printHelp = true;
-      }
-
-      if (printHelp) {
         HelpFormatter formatter = new HelpFormatter();
-        formatter.printHelp(getName() + " <table name> <table path> <number
of shards>", options);
+        formatter.printHelp(help(), options);
         return;
       }
 
@@ -105,7 +100,7 @@ public class CreateCommand extends Command {
 
   @Override
   public String help() {
-    return "create the named table, args; tableuri tablename shardcount";
+    return "create <table name> <table path> <number of shards>";
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java
----------------------------------------------------------------------
diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java
index 9c803d3..15f1ab5 100644
--- a/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java
+++ b/src/blur-shell/src/main/java/org/apache/blur/shell/DescribeCommand.java
@@ -96,7 +96,7 @@ public class DescribeCommand extends TableSingleArgumentCommand {
 
   @Override
   public String help() {
-    return "describe the named table, args; tablename";
+    return "describe <tablename>";
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java
----------------------------------------------------------------------
diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java
index 3b9d58e..32f9b13 100644
--- a/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java
+++ b/src/blur-shell/src/main/java/org/apache/blur/shell/DisableTableCommand.java
@@ -50,7 +50,7 @@ public class DisableTableCommand extends TableSingleArgumentCommand {
 
   @Override
   public String help() {
-    return "change status of the named table, args; tablename";
+    return "disable <tablename>";
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/d709ddb8/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java
----------------------------------------------------------------------
diff --git a/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java b/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java
index 0664fe4..86da060 100644
--- a/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java
+++ b/src/blur-shell/src/main/java/org/apache/blur/shell/EnableTableCommand.java
@@ -50,7 +50,7 @@ public class EnableTableCommand extends TableSingleArgumentCommand {
 
   @Override
   public String help() {
-    return "change status of the named table, args; tablename";
+    return "enable <tablename>";
   }
 
   @Override


Mime
View raw message