accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1434881 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Date Thu, 17 Jan 2013 20:01:44 GMT
Author: kturner
Date: Thu Jan 17 20:01:44 2013
New Revision: 1434881

URL: http://svn.apache.org/viewvc?rev=1434881&view=rev
Log:
ACCUMULO-956 fixed some issues and added unit test for transforming iterator

Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java?rev=1434881&r1=1434880&r2=1434881&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/KeyTransformingIterator.java
Thu Jan 17 20:01:44 2013
@@ -279,26 +279,40 @@ abstract public class KeyTransformingIte
    * @return {@code true} if the key is visible or iterator is not scanning, and {@code false}
if not
    */
   protected boolean canSee(Key key) {
-    // Ensure that the visibility (which could have been transformed) parses.
+    // Ensure that the visibility (which could have been transformed) parses. Must always
do this check, even if visibility is not evaluated.
     ByteSequence visibility = key.getColumnVisibilityData();
-    ColumnVisibility colVis = (ColumnVisibility) parsedVisibilitiesCache.get(visibility);
-    if (colVis == null) {
+    ColumnVisibility colVis = null;
+    Boolean parsed = (Boolean) parsedVisibilitiesCache.get(visibility);
+    if (parsed == null) {
       try {
         colVis = new ColumnVisibility(visibility.toArray());
+        parsedVisibilitiesCache.put(visibility, Boolean.TRUE);
       } catch (BadArgumentException e) {
-        log.error("Transformation produced an invalid visibility: " + visibility);
-        throw e;
+        log.error("Parse error after transformation : " + visibility);
+        parsedVisibilitiesCache.put(visibility, Boolean.FALSE);
+        if (scanning) {
+          return false;
+        } else {
+          throw e;
+        }
       }
+    } else if (!parsed) {
+      if (scanning)
+        return false;
+      else
+        throw new IllegalStateException();
     }
     
     Boolean visible = canSeeColumnFamily(key);
     
-    if (!scanning || !visible || ve == null || visibleCache == null)
+    if (!scanning || !visible || ve == null || visibleCache == null || visibility.length()
== 0)
       return visible;
 
     visible = (Boolean) visibleCache.get(visibility);
     if (visible == null) {
       try {
+        if (colVis == null)
+          colVis = new ColumnVisibility(visibility.toArray());
         visible = ve.evaluate(colVis);
         visibleCache.put(visibility, visible);
       } catch (VisibilityParseException e) {

Modified: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java?rev=1434881&r1=1434880&r2=1434881&view=diff
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
(original)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/KeyTransformingIteratorTest.java
Thu Jan 17 20:01:44 2013
@@ -16,18 +16,24 @@
  */
 package org.apache.accumulo.core.iterators.user;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import org.apache.accumulo.core.client.BatchScanner;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
@@ -43,9 +49,9 @@ import org.apache.accumulo.core.data.Par
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.WrappingIterator;
-import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.hadoop.io.Text;
@@ -165,6 +171,20 @@ public class KeyTransformingIteratorTest
   }
   
   @Test
+  public void testCreatingIllegalVisbility() throws Exception {
+    // illegal visibility created by transform should be filtered on scan, even if evaluation
is done
+    IteratorSetting cfg = new IteratorSetting(21, IllegalVisKeyTransformingIterator.class);
+    cfg.setName("keyTransformIter");
+    scanner.addScanIterator(cfg);
+    checkExpected(new TreeMap<Key,Value>());
+    
+    // ensure illegal vis is supressed when evaluations is done
+    scanner.removeScanIterator("keyTransformIter");
+    setUpTransformIterator(IllegalVisKeyTransformingIterator.class);
+    checkExpected(new TreeMap<Key,Value>());
+  }
+
+  @Test
   public void testRangeStart() throws Exception {
     setUpTransformIterator(ColVisReversingKeyTransformingIterator.class);
     scanner.setRange(new Range(new Key("row1", "cf2", "cq2", "vis1"), true, new Key("row1",
"cf2", "cq3"), false));
@@ -255,7 +275,44 @@ public class KeyTransformingIteratorTest
   
   @Test
   public void testDeepCopy() throws Exception {
+    MockInstance instance = new MockInstance("test");
+    Connector connector = instance.getConnector("user", "password");
+    
+    connector.tableOperations().create("shard_table");
+    
+    BatchWriter bw = connector.createBatchWriter("shard_table", new BatchWriterConfig());
+    
+    ColumnVisibility vis1 = new ColumnVisibility("vis1");
+    ColumnVisibility vis3 = new ColumnVisibility("vis3");
+    
+    Mutation m1 = new Mutation("shard001");
+    m1.put("foo", "doc02", vis1, "");
+    m1.put("dog", "doc02", vis3, "");
+    m1.put("cat", "doc02", vis3, "");
     
+    m1.put("bar", "doc03", vis1, "");
+    m1.put("dog", "doc03", vis3, "");
+    m1.put("cat", "doc03", vis3, "");
+    
+    bw.addMutation(m1);
+    bw.close();
+
+    BatchScanner bs = connector.createBatchScanner("shard_table", authorizations, 1);
+    
+    bs.addScanIterator(new IteratorSetting(21, ColVisReversingKeyTransformingIterator.class));
+    IteratorSetting iicfg = new IteratorSetting(22, IntersectingIterator.class);
+    IntersectingIterator.setColumnFamilies(iicfg, new Text[] {new Text("foo"), new Text("dog"),
new Text("cat")});
+    bs.addScanIterator(iicfg);
+    bs.setRanges(Collections.singleton(new Range()));
+    
+    Iterator<Entry<Key,Value>> iter = bs.iterator();
+    assertTrue(iter.hasNext());
+    Key docKey = iter.next().getKey();
+    assertEquals("shard001", docKey.getRowData().toString());
+    assertEquals("doc02", docKey.getColumnQualifierData().toString());
+    assertFalse(iter.hasNext());
+
+    bs.close();
   }
   
   @Test
@@ -303,6 +360,34 @@ public class KeyTransformingIteratorTest
     checkExpected(expected);
   }
   
+  @Test
+  public void testCompactionAndIllegalVisibility() throws Exception {
+    setUpTransformIterator(IllegalVisCompactionKeyTransformingIterator.class);
+    try {
+      checkExpected(new TreeMap<Key,Value>());
+      assertTrue(false);
+    } catch (Exception e) {
+      
+    }
+  }
+
+  @Test
+  public void testDupes() throws Exception {
+    setUpTransformIterator(DupeTransformingIterator.class);
+    
+    int count = 0;
+    for (Entry<Key,Value> entry : scanner) {
+      Key key = entry.getKey();
+      assertEquals("cf1", key.getColumnFamily().toString());
+      assertEquals("cq1", key.getColumnQualifier().toString());
+      assertEquals("", key.getColumnVisibility().toString());
+      assertEquals(5l, key.getTimestamp());
+      count++;
+    }
+    
+    assertEquals(81, count);
+  }
+
   private Key createDeleteKey(String row, String colFam, String colQual, String colVis, long
timestamp) {
     Key key = new Key(row, colFam, colQual, colVis, timestamp);
     key.setDeleted(true);
@@ -412,6 +497,21 @@ public class KeyTransformingIteratorTest
     };
   }
   
+  public static class DupeTransformingIterator extends KeyTransformingIterator {
+    @Override
+    protected Key transformKey(Key originalKey) {
+      Key ret = replaceKeyParts(originalKey, new Text("cf1"), new Text("cq1"), new Text(""));
+      ret.setTimestamp(5);
+      return ret;
+    };
+    
+    @Override
+    protected PartialKey getKeyPrefix() {
+      return PartialKey.ROW;
+    }
+    
+  }
+
   public static abstract class ReversingKeyTransformingIterator extends KeyTransformingIterator
{
     @Override
     protected Key transformKey(Key originalKey) {
@@ -462,6 +562,26 @@ public class KeyTransformingIteratorTest
     }
   }
   
+  public static class IllegalVisKeyTransformingIterator extends KeyTransformingIterator {
+    @Override
+    protected PartialKey getKeyPrefix() {
+      return PartialKey.ROW_COLFAM_COLQUAL;
+    }
+    
+    @Override
+    protected Key transformKey(Key originalKey) {
+      return new Key(originalKey.getRow(), originalKey.getColumnFamily(), originalKey.getColumnQualifier(),
new Text("A&|||"), originalKey.getTimestamp());
+    }
+  }
+
+  public static class IllegalVisCompactionKeyTransformingIterator extends IllegalVisKeyTransformingIterator
{
+    @Override
+    public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String>
options, IteratorEnvironment env) throws IOException {
+      env = new MajCIteratorEnvironmentAdapter(env);
+      super.init(source, options, env);
+    }
+  }
+
   public static class BadVisKeyTransformingIterator extends KeyTransformingIterator {
     @Override
     protected PartialKey getKeyPrefix() {



Mime
View raw message