accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From afu...@apache.org
Subject svn commit: r1355044 - in /accumulo/branches/ACCUMULO-652/core/src: main/java/org/apache/accumulo/core/file/rfile/ main/java/org/apache/accumulo/core/iterators/predicates/ test/java/org/apache/accumulo/core/file/rfile/
Date Thu, 28 Jun 2012 15:30:31 GMT
Author: afuchs
Date: Thu Jun 28 15:30:29 2012
New Revision: 1355044

URL: http://svn.apache.org/viewvc?rev=1355044&view=rev
Log:
ACCUMULO-652 fixed stat aggregation at higher levels, fixed some formatting, and fixed timestamp
filtering of index blocks in the IndexIterator

Modified:
    accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
    accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
    accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
    accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java

Modified: accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
(original)
+++ accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/MultiLevelIndex.java
Thu Jun 28 15:30:29 2012
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.file.blo
 import org.apache.accumulo.core.file.blockfile.BlockFileReader;
 import org.apache.accumulo.core.file.blockfile.BlockFileWriter;
 import org.apache.accumulo.core.file.rfile.bcfile.Utils;
+import org.apache.accumulo.core.iterators.predicates.ColumnVisibilityPredicate;
 import org.apache.accumulo.core.iterators.predicates.TimestampRangePredicate;
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.hadoop.io.WritableComparable;
@@ -406,7 +407,7 @@ public class MultiLevelIndex {
       writer.close(out);
     }
   }
-
+  
   public static class Writer {
     private int threshold;
     
@@ -424,8 +425,7 @@ public class MultiLevelIndex {
       levels = new ArrayList<IndexBlock>();
     }
     
-    private void add(int level, Key key, BlockStats blockStats, long offset, long compressedSize,
long rawSize, boolean last, int version)
-        throws IOException {
+    private void add(int level, Key key, BlockStats blockStats, long offset, long compressedSize,
long rawSize, boolean last, int version) throws IOException {
       if (level == levels.size()) {
         levels.add(new IndexBlock(level, 0));
       }
@@ -443,7 +443,7 @@ public class MultiLevelIndex {
         iblock.write(out);
         out.close();
         
-        add(level + 1, key, blockStats, out.getStartPos(), out.getCompressedSize(), out.getRawSize(),
last, version);
+        add(level + 1, key, iblock.blockStats, out.getStartPos(), out.getCompressedSize(),
out.getRawSize(), last, version);
         
         if (last)
           levels.set(level, null);
@@ -501,9 +501,11 @@ public class MultiLevelIndex {
     class IndexIterator implements Iterator<IndexEntry> {
       private Stack<StackEntry> position = new Stack<StackEntry>();
       private final TimestampRangePredicate timestampFilter;
+      private final ColumnVisibilityPredicate columnVisibilityPredicate;
       
-      private IndexIterator(TimestampRangePredicate timestampFilter, Key lookupKey) {
+      private IndexIterator(TimestampRangePredicate timestampFilter, ColumnVisibilityPredicate
columnVisibilityPredicate, Key lookupKey) {
         this.timestampFilter = timestampFilter;
+        this.columnVisibilityPredicate = columnVisibilityPredicate;
         try {
           seek(lookupKey);
         } catch (IOException e) {
@@ -512,10 +514,10 @@ public class MultiLevelIndex {
       }
       
       private final boolean checkFilterIndexEntry(IndexEntry ie) {
-        if(timestampFilter == null)
-        if (timestampFilter != null && (ie.blockStats.maxTimestamp < timestampFilter.startTimestamp
|| ie.blockStats.minTimestamp > timestampFilter.endTimestamp)) {
+        if(timestampFilter != null && (ie.blockStats.maxTimestamp < timestampFilter.startTimestamp
|| ie.blockStats.minTimestamp > timestampFilter.endTimestamp))
+          return false;
+        if(columnVisibilityPredicate != null && ie.blockStats.minimumVisibility !=
null && ie.blockStats.minimumVisibility.evaluate(columnVisibilityPredicate.auths)
== false)
           return false;
-        }
         return true;
       }
       
@@ -532,7 +534,6 @@ public class MultiLevelIndex {
             }
           });
           
-          
           if (pos < 0) {
             pos = (pos * -1) - 1;
           } else if (pos < top.block.getKeyIndex().size()) {
@@ -542,14 +543,13 @@ public class MultiLevelIndex {
             }
           }
           
-
           IndexEntry ie = null;
           List<IndexEntry> index = top.block.getIndex();
           
-          if(pos > 0)
-          {
-            // look backwards to find any initial previousEntry that might match the timestamp
range such that no entry within the given timestamp range is between the seeked key and the
previousKey
-            previousEntry = index.get(pos-1);
+          if (pos > 0) {
+            // look backwards to find any initial previousEntry that might match the timestamp
range such that no entry within the given timestamp range is
+            // between the seeked key and the previousKey
+            previousEntry = index.get(pos - 1);
             // TODO: find the offset for this block
             previousIndex = Integer.MIN_VALUE;
           }
@@ -562,7 +562,6 @@ public class MultiLevelIndex {
             pos++;
           }
           
-          
           if (pos == index.size()) {
             position.pop();
             goToNext();
@@ -581,7 +580,6 @@ public class MultiLevelIndex {
       }
       
       private void goToNext() throws IOException {
-        int numSkippedBlocks = 0;
         // traverse the index tree forwards
         while (position.isEmpty() == false) {
           StackEntry top = position.peek();
@@ -590,7 +588,6 @@ public class MultiLevelIndex {
           while (top.offset < index.size()) {
             if (checkFilterIndexEntry(index.get(top.offset)))
               break;
-            numSkippedBlocks++;
             top.offset++;
           }
           if (top.offset == index.size()) {
@@ -651,8 +648,6 @@ public class MultiLevelIndex {
         return nextEntry;
       }
       
-      private int blocksReturned = 0;
-      
       public IndexEntry next() {
         prepNext();
         previousEntry = nextEntry;
@@ -697,7 +692,7 @@ public class MultiLevelIndex {
     }
     
     IndexIterator lookup(Key key) throws IOException {
-      return new IndexIterator(timestampRange, key);
+      return new IndexIterator(timestampRange, columnVisibilityPredicate, key);
     }
     
     public void readFields(DataInput in) throws IOException {
@@ -751,7 +746,7 @@ public class MultiLevelIndex {
       return rootBlock.getIndex().get(rootBlock.getIndex().size() - 1).getKey();
     }
     
-    TimestampRangePredicate timestampRange;
+    TimestampRangePredicate timestampRange = null;
     
     /**
      * @param r
@@ -759,6 +754,12 @@ public class MultiLevelIndex {
     public void setTimestampRange(TimestampRangePredicate r) {
       this.timestampRange = r;
     }
+    
+    ColumnVisibilityPredicate columnVisibilityPredicate = null;
+    
+    public void setColumnVisibilityPredicate(ColumnVisibilityPredicate columnVisibilityPredicate)
{
+      this.columnVisibilityPredicate = columnVisibilityPredicate;
+    }
   }
   
 }

Modified: accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
(original)
+++ accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
Thu Jun 28 15:30:29 2012
@@ -61,6 +61,7 @@ import org.apache.accumulo.core.iterator
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.Predicate;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.predicates.ColumnVisibilityPredicate;
 import org.apache.accumulo.core.iterators.predicates.TimestampRangePredicate;
 import org.apache.accumulo.core.iterators.system.HeapIterator;
 import org.apache.accumulo.core.security.ColumnVisibility;
@@ -776,6 +777,7 @@ public class RFile {
     }
     
     private TimestampRangePredicate timestampRange;
+    private ColumnVisibilityPredicate columnVisibilityPredicate;
     private boolean filterChanged = false;
 
     /* (non-Javadoc)
@@ -795,6 +797,12 @@ public class RFile {
           timestampRange = p;
         index.setTimestampRange(timestampRange);
       }
+      else if(filter instanceof ColumnVisibilityPredicate)
+      {
+    	  filterChanged = true;
+    	  columnVisibilityPredicate = (ColumnVisibilityPredicate)filter;
+    	  index.setColumnVisibilityPredicate(columnVisibilityPredicate);
+      }
       else
       {
         throw new RuntimeException("yikes, not yet implemented");

Modified: accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
(original)
+++ accumulo/branches/ACCUMULO-652/core/src/main/java/org/apache/accumulo/core/iterators/predicates/TimestampRangePredicate.java
Thu Jun 28 15:30:29 2012
@@ -51,4 +51,8 @@ public class TimestampRangePredicate imp
     return timestamp >= startTimestamp && timestamp <= endTimestamp;
   }
   
+  @Override
+  public String toString() {
+    return "{"+startTimestamp+"-"+endTimestamp+"}";
+  }
 }

Modified: accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java?rev=1355044&r1=1355043&r2=1355044&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java
(original)
+++ accumulo/branches/ACCUMULO-652/core/src/test/java/org/apache/accumulo/core/file/rfile/TimestampFilterTest.java
Thu Jun 28 15:30:29 2012
@@ -46,7 +46,7 @@ public class TimestampFilterTest {
   @Test
   public void testRFileTimestampFiltering() throws Exception {
     // TODO create an RFile with increasing timestamp and random key order
-    Predicate<Key,Value> timeRange = new TimestampRangePredicate(100, 110);
+    Predicate<Key,Value> timeRange = new TimestampRangePredicate(73, 117);
     int expected = 0;
     Random r = new Random();
     Configuration conf = new Configuration();



Mime
View raw message