accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1222413 - in /incubator/accumulo/trunk/contrib/accumulo_sample: ./ query/src/main/java/iterator/ query/src/main/java/logic/ query/src/main/java/sample/query/
Date Thu, 22 Dec 2011 19:59:31 GMT
Author: ecn
Date: Thu Dec 22 19:59:30 2011
New Revision: 1222413

URL: http://svn.apache.org/viewvc?rev=1222413&view=rev
Log:
ACCUMULO-230
Fixed key decoding in iterators.
Fixed restart of query in AndIterator
Fixed the detection of scan/seek in OptimizedQueryIterator
Added code to skip over the Field Index within the tablet

Modified:
    incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java
    incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml (original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/pom.xml Thu Dec 22 19:59:30 2011
@@ -187,7 +187,7 @@
     <version.lucene-wikipedia>3.0.2</version.lucene-wikipedia>
     <version.protobuf>2.3.0</version.protobuf>
     <version.googlecollections>1.0</version.googlecollections>
-    <version.libthrift>0.8.0</version.libthrift>
+    <version.libthrift>0.6.1</version.libthrift>
     <version.zookeeper>3.3.1</version.zookeeper>
     <version.minlog>1.2</version.minlog>
   </properties>

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AbstractEvaluatingIterator.java
Thu Dec 22 19:59:30 2011
@@ -25,11 +25,6 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.jexl2.parser.ParseException;
-import org.apache.log4j.Logger;
-
-import parser.EventFields;
-import parser.QueryEvaluator;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.PartialKey;
@@ -38,6 +33,11 @@ import org.apache.accumulo.core.data.Val
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.OptionDescriber;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.commons.jexl2.parser.ParseException;
+import org.apache.log4j.Logger;
+
+import parser.EventFields;
+import parser.QueryEvaluator;
 
 import com.esotericsoftware.kryo.Kryo;
 
@@ -66,7 +66,7 @@ public abstract class AbstractEvaluating
   public static final String UNEVALUTED_EXPRESSIONS = "unevaluated.expressions";
   
   private PartialKey comparator = null;
-  private SortedKeyValueIterator<Key,Value> iterator;
+  protected SortedKeyValueIterator<Key,Value> iterator;
   private Key currentKey = new Key();
   private Key returnKey;
   private Value returnValue;
@@ -118,8 +118,9 @@ public abstract class AbstractEvaluating
    * 
    * @param key
    * @return
+   * @throws IOException
    */
-  public abstract boolean isKeyAccepted(Key key);
+  public abstract boolean isKeyAccepted(Key key) throws IOException;
   
   /**
    * Reset state.

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/AndIterator.java
Thu Dec 22 19:59:30 2011
@@ -140,7 +140,7 @@ public class AndIterator implements Sort
     int idx = 0;
     String sKey = key.getColumnQualifier().toString();
     
-    idx = sKey.lastIndexOf("\0");
+    idx = sKey.indexOf("\0");
     return sKey.substring(idx + 1);
   }
   
@@ -781,15 +781,16 @@ public class AndIterator implements Sort
       Key sourceKey;
       if (range.getStartKey() != null) {
         // Build a key with the DocID if one is given
-        if (range.getStartKey().getColumnQualifier() != null) {
+        if (range.getStartKey().getColumnFamily() != null) {
           sourceKey = buildKey(getPartition(range.getStartKey()), (sources[i].dataLocation
== null) ? nullText : sources[i].dataLocation,
-              (sources[i].term == null) ? nullText : new Text(sources[i].term + "\0" + range.getStartKey().getColumnQualifier()));
+              (sources[i].term == null) ? nullText : new Text(sources[i].term + "\0" + range.getStartKey().getColumnFamily()));
         } // Build a key with just the term.
         else {
           sourceKey = buildKey(getPartition(range.getStartKey()), (sources[i].dataLocation
== null) ? nullText : sources[i].dataLocation,
               (sources[i].term == null) ? nullText : sources[i].term);
         }
-        
+        if (!range.isStartKeyInclusive())
+          sourceKey = sourceKey.followingKey(PartialKey.ROW_COLFAM_COLQUAL);
         sources[i].iter.seek(new Range(sourceKey, true, null, false), seekColumnFamilies,
inclusive);
       } else {
         sources[i].iter.seek(range, seekColumnFamilies, inclusive);
@@ -889,7 +890,7 @@ public class AndIterator implements Sort
         return hasTop();
       } else {
         // need to check uid
-        String myUid = getUID(this.topKey);
+        String myUid = this.topKey.getColumnQualifier().toString();
         String jumpUid = getUID(jumpKey);
         if (log.isDebugEnabled()) {
           if (myUid == null) {

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/BooleanLogicIterator.java
Thu Dec 22 19:59:30 2011
@@ -1204,16 +1204,10 @@ public class BooleanLogicIterator implem
   }
   
   private String getEventKeyUid(Key k) {
-    String uid = null;
     if (k == null || k.getColumnFamily() == null) {
-      return uid;
+      return null;
     } else {
-      String[] parts = k.getColumnFamily().toString().split("\0");
-      if (parts.length < 2) {
-        return uid;
-      } else {
-        return parts[parts.length - 1];
-      }
+      return k.getColumnFamily().toString();
     }
   }
   
@@ -1221,7 +1215,7 @@ public class BooleanLogicIterator implem
     try {
       int idx = 0;
       String sKey = k.getColumnQualifier().toString();
-      idx = sKey.lastIndexOf("\0");
+      idx = sKey.indexOf("\0");
       return sKey.substring(idx + 1);
     } catch (Exception e) {
       return null;
@@ -1865,6 +1859,8 @@ public class BooleanLogicIterator implem
         }
       }
       
+      log.debug("overallRange " + overallRange + " topKey " + this.root.getTopKey() + " contains
" + overallRange.contains(this.root.getTopKey()));
+
       if (overallRange.contains(this.root.getTopKey())) {
         setTopKey(this.root.getTopKey());
       } else {

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/EvaluatingIterator.java
Thu Dec 22 19:59:30 2011
@@ -16,14 +16,22 @@
  */
 package iterator;
 
-import parser.EventFields;
-import parser.EventFields.FieldValue;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.hadoop.io.Text;
+
+import parser.EventFields;
+import parser.EventFields.FieldValue;
 
 public class EvaluatingIterator extends AbstractEvaluatingIterator {
   
@@ -77,9 +85,14 @@ public class EvaluatingIterator extends 
    * Don't accept this key if the colf starts with 'fi'
    */
   @Override
-  public boolean isKeyAccepted(Key key) {
+  public boolean isKeyAccepted(Key key) throws IOException {
     if (key.getColumnFamily().toString().startsWith("fi")) {
-      return false;
+      Key copy = new Key(key.getRow(), new Text("fi\01"));
+      Collection<ByteSequence> columnFamilies = Collections.emptyList();
+      this.iterator.seek(new Range(copy, copy), columnFamilies, true);
+      if (this.iterator.hasTop())
+        return isKeyAccepted(this.iterator.getTopKey());
+      return true;
     }
     return true;
   }

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OptimizedQueryIterator.java
Thu Dec 22 19:59:30 2011
@@ -22,15 +22,15 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 
-import org.apache.log4j.Logger;
-
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.OptionDescriber;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.log4j.Logger;
 
 /**
  * This iterator internally uses the BooleanLogicIterator to find event UIDs in the field
index portion of the partition and uses the EvaluatingIterator to
@@ -44,6 +44,7 @@ public class OptimizedQueryIterator impl
   private Key key = null;
   private Value value = null;
   private boolean eventSpecificRange = false;
+  private long seekTime = 0;
   
   public IteratorOptions describeOptions() {
     Map<String,String> options = new HashMap<String,String>();
@@ -139,7 +140,9 @@ public class OptimizedQueryIterator impl
         // If the index has a match, then seek the event to the key
         if (index.hasTop()) {
           Key eventKey = index.getTopKey();
-          Range eventRange = new Range(eventKey.getRow());
+          Key endKey = eventKey.followingKey(PartialKey.ROW_COLFAM);
+          Key startKey = new Key(eventKey.getRow(), eventKey.getColumnFamily());
+          Range eventRange = new Range(startKey, endKey);
           HashSet<ByteSequence> cf = new HashSet<ByteSequence>();
           cf.add(eventKey.getColumnFamilyData());
           event.seek(eventRange, cf, true);
@@ -164,7 +167,7 @@ public class OptimizedQueryIterator impl
       log.debug("seek, range:" + range);
     }
     // Test the range to see if it is event specific.
-    if (null != range.getStartKey() && range.getStartKey().getColumnFamily() != null
&& !range.getStartKey().getColumnFamily().toString().equals("")) {
+    if (null != range.getEndKey() && range.getEndKey().getColumnFamily() != null
&& range.getEndKey().getColumnFamily().getLength() != 0) {
       if (log.isDebugEnabled()) {
         log.debug("Jumping straight to the event");
       }

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/iterator/OrIterator.java
Thu Dec 22 19:59:30 2011
@@ -23,9 +23,6 @@ import java.util.Iterator;
 import java.util.Map;
 import java.util.PriorityQueue;
 
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.PartialKey;
@@ -33,6 +30,8 @@ import org.apache.accumulo.core.data.Ran
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
 
 /**
  * An iterator that handles "OR" query constructs on the server side. This code has been
adapted/merged from Heap and Multi Iterators.
@@ -206,7 +205,7 @@ public class OrIterator implements Sorte
       int idx = 0;
       String sKey = key.getColumnQualifier().toString();
       
-      idx = sKey.lastIndexOf("\0");
+      idx = sKey.indexOf("\0");
       return sKey.substring(idx + 1);
     } catch (Exception e) {
       return null;

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/logic/AbstractQueryLogic.java
Thu Dec 22 19:59:30 2011
@@ -58,7 +58,6 @@ import parser.EventFields;
 import parser.EventFields.FieldValue;
 import parser.FieldIndexQueryReWriter;
 import parser.JexlOperatorConstants;
-import parser.JexlOperatorConstants;
 import parser.QueryParser;
 import parser.QueryParser.QueryTerm;
 import parser.RangeCalculator;
@@ -460,7 +459,8 @@ public abstract class AbstractQueryLogic
     
     Set<Range> ranges = new HashSet<Range>();
     Set<String> typeFilter = types;
-    Authorizations auths = new Authorizations(StringUtils.join(authorizations, "|"));
+    String array[] = authorizations.toArray(new String[0]);
+    Authorizations auths = new Authorizations(array);
     Results results = new Results();
     
     // Get the query string
@@ -785,9 +785,6 @@ public abstract class AbstractQueryLogic
           long count = 0;
           for (Entry<Key,Value> entry : bs) {
             count++;
-            if (log.isDebugEnabled()) {
-              log.debug("Key: " + entry.getKey());// + "\nValue: " + entry.getValue() + "\n");
-            }
             // The key that is returned by the EvaluatingIterator is not the same key that
is in
             // the table. The value that is returned by the EvaluatingIterator is a kryo
             // serialized EventFields object.

Modified: incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java?rev=1222413&r1=1222412&r2=1222413&view=diff
==============================================================================
--- incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java
(original)
+++ incubator/accumulo/trunk/contrib/accumulo_sample/query/src/main/java/sample/query/Query.java
Thu Dec 22 19:59:30 2011
@@ -192,9 +192,12 @@ public class Query implements IQuery {
     
     // Create list of auths
     List<String> authorizations = new ArrayList<String>();
-    for (String a : auths.split(","))
-      authorizations.add(a);
-    
+    if (auths == null || "".equals(auths)) {
+      authorizations.add("all");
+    } else {
+      for (String a : auths.split(","))
+        authorizations.add(a);
+    }
     ContentLogic table = new ContentLogic();
     table.setTableName(tableName);
     return table.runQuery(connector, query, authorizations);



Mime
View raw message