incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject [2/4] git commit: Added a CLI and limit to Bql.
Date Wed, 10 Oct 2012 02:01:29 GMT
Added a CLI and limit to Bql.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5fc3afaa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5fc3afaa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5fc3afaa

Branch: refs/heads/new-api-prototype
Commit: 5fc3afaa3ccae2c1690942cb5bf5aa5ce50517e8
Parents: 3fbd7f5
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Tue Oct 9 19:58:33 2012 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Tue Oct 9 19:58:33 2012 -0400

----------------------------------------------------------------------
 .../main/java/org/apache/blur/core/ServerHdfs.java |   17 ++-
 .../src/main/java/org/apache/blur/core/Util.java   |   19 +++
 .../blur/core/sessions/RunningQuerySession.java    |   48 ++++---
 .../java/org/apache/blur/parser/BqlParser.java     |    2 +-
 .../example/clients/ReadThriftClientBatch.java     |    4 +-
 .../example/clients/ReadThriftClientBatchCli.java  |  107 +++++++++++++++
 6 files changed, 168 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fc3afaa/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/ServerHdfs.java
----------------------------------------------------------------------
diff --git a/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/ServerHdfs.java
b/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/ServerHdfs.java
index 56db60e..2487e3b 100644
--- a/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/ServerHdfs.java
+++ b/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/ServerHdfs.java
@@ -23,7 +23,6 @@ import java.lang.management.RuntimeMXBean;
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -76,7 +75,6 @@ public class ServerHdfs implements Iface {
   private static final int MIN_SLABSIZE = 33554432;
   private static final int MAX_SLABSIZE = 134217728;
 
-  private static final List<Tuple> EMPTY_LIST = new ArrayList<Tuple>();
   private static final long KILO = 1024;
   private static final long MEG = KILO * 1024;
   private static final long GIG = MEG * 1024;
@@ -208,12 +206,15 @@ public class ServerHdfs implements Iface {
     try {
       ReadSession session = getReadSession(querySession.getSession());
       RunningQuerySession runningQuerySession = session.getRunningQuerySession(querySession.getQueryId());
-      if (runningQuerySession.isMetaDataBeenFetched()) {
-        return EMPTY_LIST;
+      List<Tuple> results = new ArrayList<Tuple>();
+      for (int i = 0; i < batchSize; i++) {
+        Tuple tuple = runningQuerySession.nextMetaData();
+        if (tuple == null) {
+          break;
+        }
+        results.add(tuple);
       }
-      Tuple tuple = new Tuple();
-      tuple.addToAttributes(Util.newAttribute("totalResults", runningQuerySession.getTotalHits()));
-      return Arrays.asList(tuple);
+      return results;
     } catch (Throwable t) {
       throw Util.wrapThrowable(t);
     }
@@ -226,7 +227,7 @@ public class ServerHdfs implements Iface {
       RunningQuerySession runningQuerySession = session.getRunningQuerySession(querySession.getQueryId());
       List<Tuple> results = new ArrayList<Tuple>();
       for (int i = 0; i < batchSize; i++) {
-        Tuple tuple = runningQuerySession.nextDocument();
+        Tuple tuple = runningQuerySession.nextData();
         if (tuple == null) {
           break;
         }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fc3afaa/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/Util.java
----------------------------------------------------------------------
diff --git a/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/Util.java b/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/Util.java
index e24412d..8b6abb6 100644
--- a/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/Util.java
+++ b/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/Util.java
@@ -45,6 +45,17 @@ public class Util {
     attribute.setType(TYPE.INT);
     return attribute;
   }
+  
+  public static Attribute newAttribute(String name, double value) {
+    ByteBuffer buffer = ByteBuffer.allocate(8);
+    buffer.putDouble(value);
+    buffer.flip();
+    Attribute attribute = new Attribute();
+    attribute.setName(name);
+    attribute.setValue(buffer.array());
+    attribute.setType(TYPE.DOUBLE);
+    return attribute;
+  }
 
   public static Attribute toAttribute(IndexableField fieldable) {
     return newAttribute(fieldable.name(), fieldable.stringValue());
@@ -79,6 +90,9 @@ public class Util {
       case INT:
         s += toInt(attribute.getValue());
         break;
+      case DOUBLE:
+        s += toDouble(attribute.getValue());
+        break;
       default:
         throw new RuntimeException("Type [" + attribute.getType() + "] not supported.");
       }
@@ -86,6 +100,10 @@ public class Util {
     return "{" + s + "}";
   }
 
+  private static double toDouble(byte[] bs) {
+    return ByteBuffer.wrap(bs).getDouble();
+  }
+
   public static int toInt(byte[] bs) {
     return ByteBuffer.wrap(bs).getInt();
   }
@@ -99,4 +117,5 @@ public class Util {
     return new BlurException(t.getMessage(), stackTraceStr);
   }
 
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fc3afaa/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/sessions/RunningQuerySession.java
----------------------------------------------------------------------
diff --git a/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/sessions/RunningQuerySession.java
b/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/sessions/RunningQuerySession.java
index a195c1b..9336d70 100644
--- a/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/sessions/RunningQuerySession.java
+++ b/src/blur-new-api-prototype/src/main/java/org/apache/blur/core/sessions/RunningQuerySession.java
@@ -9,6 +9,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
 import org.apache.blur.core.TupleStoredFieldVisitor;
+import org.apache.blur.core.Util;
 import org.apache.blur.parser.BqlParser;
 import org.apache.blur.parser.BqlParser.OrderBy;
 import org.apache.blur.parser.BqlParser.QueryType;
@@ -27,7 +28,7 @@ public class RunningQuerySession {
 
   private IndexSearcher searcher;
   private TopDocs topDocs;
-  private int totalHits;
+  private int count;
   private Query query;
   private int position = 0;
   private int totalPosition = 0;
@@ -43,6 +44,9 @@ public class RunningQuerySession {
   private TupleStoredFieldVisitor fieldVistor;
   private Sort sort;
   private long limit;
+  private Query queryRewrite;
+  private double rewriteTime;
+  private double queryTime;
 
   public RunningQuerySession(IndexReader reader, String sessionId, String queryId) {
     searcher = new IndexSearcher(reader);
@@ -81,20 +85,23 @@ public class RunningQuerySession {
     }
   }
 
-  public boolean isMetaDataBeenFetched() {
-    return metaData;
-  }
-
-  public int getTotalHits() {
-    metaData = true;
-    return totalHits;
+  public Tuple nextMetaData() throws IOException {
+    if (!metaData) {
+      Tuple tuple = new Tuple();
+      tuple.addToAttributes(Util.newAttribute("count", count));
+      tuple.addToAttributes(Util.newAttribute("rewriteTime", rewriteTime));
+      tuple.addToAttributes(Util.newAttribute("queryTime", queryTime));
+      metaData = true;
+      return tuple;
+    }
+    return null;
   }
 
-  public Tuple nextDocument() throws IOException {
+  public Tuple nextData() throws IOException {
     if (totalPosition >= limit) {
       return null;
     }
-    if (totalPosition >= totalHits) {
+    if (totalPosition >= count) {
       return null;
     }
     try {
@@ -113,12 +120,18 @@ public class RunningQuerySession {
   }
 
   private void execute() throws IOException {
+    long s = System.nanoTime();
+    queryRewrite = searcher.rewrite(query);
+    long m = System.nanoTime();
     if (sort == null) {
-      topDocs = searcher.search(query, fetch);
+      topDocs = searcher.search(queryRewrite, fetch);
     } else {
-      topDocs = searcher.search(query, fetch, sort);
+      topDocs = searcher.search(queryRewrite, fetch, sort);
     }
-    totalHits = topDocs.totalHits;
+    long e = System.nanoTime();
+    rewriteTime = (m-s) / 1000000.0;
+    queryTime = (e-m) / 1000000.0;
+    count = topDocs.totalHits;
   }
 
   private void checkToSeeIfWeCanGetAhead() throws IOException {
@@ -130,11 +143,10 @@ public class RunningQuerySession {
           @Override
           public TopDocs call() throws Exception {
             if (sort == null) {
-              return searcher.searchAfter(after, query, fetch);
+              return searcher.searchAfter(after, queryRewrite, fetch);
             } else {
-              return searcher.searchAfter(after, query, fetch, sort);
+              return searcher.searchAfter(after, queryRewrite, fetch, sort);
             }
-//            return searcher.searchAfter(after, query, fetch);
           }
         });
       }
@@ -154,9 +166,9 @@ public class RunningQuerySession {
     } else {
       pagedBefore = true;
       if (sort == null) {
-        topDocs = searcher.searchAfter(scoreDoc, query, fetch);
+        topDocs = searcher.searchAfter(scoreDoc, queryRewrite, fetch);
       } else {
-        topDocs = searcher.searchAfter(scoreDoc, query, fetch, sort);
+        topDocs = searcher.searchAfter(scoreDoc, queryRewrite, fetch, sort);
       }
     }
     position = 0;

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fc3afaa/src/blur-new-api-prototype/src/main/java/org/apache/blur/parser/BqlParser.java
----------------------------------------------------------------------
diff --git a/src/blur-new-api-prototype/src/main/java/org/apache/blur/parser/BqlParser.java
b/src/blur-new-api-prototype/src/main/java/org/apache/blur/parser/BqlParser.java
index 832a734..c602072 100644
--- a/src/blur-new-api-prototype/src/main/java/org/apache/blur/parser/BqlParser.java
+++ b/src/blur-new-api-prototype/src/main/java/org/apache/blur/parser/BqlParser.java
@@ -77,7 +77,7 @@ public class BqlParser {
         return true;
       }
     }
-    return limitTokenFound;
+    return true;
   }
 
   private boolean parseOrderBy(String query) throws BqlException {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fc3afaa/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatch.java
----------------------------------------------------------------------
diff --git a/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatch.java
b/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatch.java
index d6ce88b..ebaacaf 100644
--- a/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatch.java
+++ b/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatch.java
@@ -38,9 +38,9 @@ public class ReadThriftClientBatch {
         Session session = client.openReadSession();
         // QuerySession querySession = client.executeQuery(session,
         // "select id0,id1 from table1 order by desc id0");
-        QuerySession querySession = client.executeQuery(session, "select id0,id1 from table1
limit 100");
         // QuerySession querySession = client.executeQuery(session,
-        // "select id0,id1,id2 as id_othername from table1 where id1:86184300004670* order
by desc id0");
+        // "select id0,id1 from table1 limit 100");
+        QuerySession querySession = client.executeQuery(session, "select id0,id1,id2 as id_othername
from table1 where id1:7618430000467068610~ order by desc id0");
         while (true) {
           List<Tuple> metaDataResults = client.nextMetaDataResults(querySession, 100);
           if (metaDataResults.isEmpty()) {

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fc3afaa/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatchCli.java
----------------------------------------------------------------------
diff --git a/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatchCli.java
b/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatchCli.java
new file mode 100644
index 0000000..bbcde98
--- /dev/null
+++ b/src/blur-new-api-prototype/src/main/java/org/apache/blur/proto/example/clients/ReadThriftClientBatchCli.java
@@ -0,0 +1,107 @@
+package org.apache.blur.proto.example.clients;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.List;
+
+import org.apache.blur.core.Util;
+import org.apache.blur.thrift.ClientManager;
+import org.apache.blur.thrift.Command;
+import org.apache.blur.thrift.generated.BlurException;
+import org.apache.blur.thrift.generated.BlurTuple.Client;
+import org.apache.blur.thrift.generated.QuerySession;
+import org.apache.blur.thrift.generated.Session;
+import org.apache.blur.thrift.generated.Tuple;
+import org.apache.thrift.TException;
+
+public class ReadThriftClientBatchCli {
+
+  public static void main(String[] args) throws BlurException, TException, IOException {
+    BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
+    String line;
+    System.out.print(">");
+    System.out.flush();
+    while ((line = reader.readLine()) != null) {
+      executeQuery(line.trim());
+      System.out.print(">");
+      System.out.flush();
+    }
+  }
+
+  private static void executeQuery(final String q) throws BlurException, TException, IOException
{
+    ClientManager.execute("localhost:9000", new Command<Void>() {
+      @Override
+      public Void call(Client client) throws BlurException, TException {
+        Session session = client.openReadSession();
+        QuerySession querySession = client.executeQuery(session, q);
+        while (true) {
+          List<Tuple> metaDataResults = client.nextMetaDataResults(querySession, 100);
+          if (metaDataResults.isEmpty()) {
+            break;
+          }
+          for (Tuple tuple : metaDataResults) {
+            System.out.println(Util.toString(tuple));
+          }
+        }
+
+        long count = 0;
+        long total = 0;
+        long start = System.nanoTime();
+        long attributeCount = 0;
+        long attributeTotal = 0;
+        while (true) {
+          long now = System.nanoTime();
+          if (start + 5000000000L < now) {
+            double seconds = (now - start) / 1000000000.0;
+            double rate = count / seconds;
+            double attributeRate = attributeCount / seconds;
+            System.out.println("Tuple count [" + total + "] at [" + rate + "/s] Attribute
Count [" + attributeTotal + "] at [" + attributeRate + "/s]");
+            start = System.nanoTime();
+            count = 0;
+            attributeCount = 0;
+          }
+          List<Tuple> results = client.nextResults(querySession, 100);
+          if (results.isEmpty()) {
+            break;
+          }
+          for (Tuple tuple : results) {
+            System.out.println(Util.toString(tuple));
+          }
+          count += results.size();
+          total += results.size();
+          int atCount = 0;
+          for (Tuple t : results) {
+            atCount += t.getAttributes().size();
+          }
+          attributeCount += atCount;
+          attributeTotal += atCount;
+        }
+        client.closeReadSession(session);
+
+        long now = System.nanoTime();
+        double seconds = (now - start) / 1000000000.0;
+        double rate = count / seconds;
+        double attributeRate = attributeCount / seconds;
+        System.out.println("Tuple count [" + total + "] at [" + rate + "/s] Attribute Count
[" + attributeTotal + "] at [" + attributeRate + "/s]");
+        return null;
+      }
+    });
+  }
+}


Mime
View raw message