accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1478404 - in /accumulo/trunk: core/src/main/java/org/apache/accumulo/core/constraints/ core/src/main/java/org/apache/accumulo/core/iterators/ core/src/test/java/org/apache/accumulo/core/constraints/ proxy/src/test/java/org/apache/accumulo/...
Date Thu, 02 May 2013 15:12:30 GMT
Author: ecn
Date: Thu May  2 15:12:30 2013
New Revision: 1478404

URL: http://svn.apache.org/r1478404
Log:
ACCUMULO-466 commit Corey Nolet's patch to add a default constraint limit the size of keys

Added:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraint.java
  (with props)
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/constraints/
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraintTest.java
  (with props)
Modified:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
    accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
    accumulo/trunk/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
    accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java

Added: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraint.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraint.java?rev=1478404&view=auto
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraint.java
(added)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraint.java
Thu May  2 15:12:30 2013
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.constraints;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.accumulo.core.data.ColumnUpdate;
+import org.apache.accumulo.core.data.Mutation;
+
+/**
+ * A constraints that limits the size of keys to 1mb.
+ */
+public class DefaultKeySizeConstraint implements Constraint {
+  
+  protected static final short MAX__KEY_SIZE_EXCEEDED_VIOLATION = 1;
+  protected static final long maxSize = 1048576; // 1MB default size
+  
+  @Override
+  public String getViolationDescription(short violationCode) {
+    
+    switch (violationCode) {
+      case MAX__KEY_SIZE_EXCEEDED_VIOLATION:
+        return "Key was larger than 1MB";
+    }
+    
+    return null;
+  }
+  
+  @Override
+  public List<Short> check(Environment env, Mutation mutation) {
+
+    List<Short> violations = new ArrayList<Short>();
+
+    for (ColumnUpdate cu : mutation.getUpdates()) {
+      int size = mutation.getRow().length;
+      size += cu.getColumnFamily().length;
+      size += cu.getColumnQualifier().length;
+      size += cu.getColumnVisibility().length;
+
+      if (size > maxSize)
+        violations.add(MAX__KEY_SIZE_EXCEEDED_VIOLATION);
+    }
+
+    return violations;
+  }
+}

Propchange: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraint.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java?rev=1478404&r1=1478403&r2=1478404&view=diff
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
(original)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/IteratorUtil.java
Thu May  2 15:12:30 2013
@@ -30,6 +30,7 @@ import java.util.TreeMap;
 import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.DefaultKeySizeConstraint;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Range;
@@ -78,7 +79,9 @@ public class IteratorUtil {
           props.put(Property.TABLE_ITERATOR_PREFIX + iterScope.name() + ".vers.opt.maxVersions",
"1");
         }
     }
-    
+
+    props.put(Property.TABLE_CONSTRAINT_PREFIX.toString() + "1", DefaultKeySizeConstraint.class.getName());
+
     return props;
   }
   

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraintTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraintTest.java?rev=1478404&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraintTest.java
(added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraintTest.java
Thu May  2 15:12:30 2013
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.constraints;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collections;
+
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+public class DefaultKeySizeConstraintTest {
+  
+  Constraint constraint = new DefaultKeySizeConstraint();
+  
+  byte[] oversized = new byte[1048577];
+  byte[] large = new byte[419430];
+  
+  @Test
+  public void testConstraint() {
+
+    // pass constraints
+    Mutation m = new Mutation("rowId");
+    m.put("colf", "colq", new Value(new byte[]{}));
+    assertEquals(Collections.emptyList(), constraint.check(null, m));
+
+    // test with row id > 1mb
+    m = new Mutation(oversized);
+    m.put("colf", "colq", new Value(new byte[]{}));
+    assertEquals(Collections.singletonList(DefaultKeySizeConstraint.MAX__KEY_SIZE_EXCEEDED_VIOLATION),
constraint.check(null, m));
+
+    // test with colf > 1mb
+    m = new Mutation("rowid");
+    m.put(new Text(oversized), new Text("colq"), new Value(new byte[]{}));
+    assertEquals(Collections.singletonList(DefaultKeySizeConstraint.MAX__KEY_SIZE_EXCEEDED_VIOLATION),
constraint.check(null, m));
+
+    // test with colf > 1mb
+    m = new Mutation("rowid");
+    m.put(new Text(oversized), new Text("colq"), new Value(new byte[]{}));
+    assertEquals(Collections.singletonList(DefaultKeySizeConstraint.MAX__KEY_SIZE_EXCEEDED_VIOLATION),
constraint.check(null, m));
+
+    // test sum of smaller sizes violates 1mb constraint
+    m = new Mutation(large);
+    m.put(new Text(large), new Text(large), new Value(new byte[] {}));
+    assertEquals(Collections.singletonList(DefaultKeySizeConstraint.MAX__KEY_SIZE_EXCEEDED_VIOLATION),
constraint.check(null, m));
+  }
+}

Propchange: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/constraints/DefaultKeySizeConstraintTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java?rev=1478404&r1=1478403&r2=1478404&view=diff
==============================================================================
--- accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java (original)
+++ accumulo/trunk/proxy/src/test/java/org/apache/accumulo/proxy/SimpleTest.java Thu May 
2 15:12:30 2013
@@ -635,7 +635,7 @@ public class SimpleTest {
       fail("constraint did not fire");
     } catch (MutationsRejectedException e) {}
     
-    client.removeConstraint(creds, TABLE_TEST, 1);
+    client.removeConstraint(creds, TABLE_TEST, 2);
     
     writerOptions = new WriterOptions();
     writerOptions.setLatencyMs(10000);
@@ -671,7 +671,7 @@ public class SimpleTest {
       fail("constraint did not fire");
     } catch (MutationsRejectedException ex) {}
     
-    client.removeConstraint(creds, TABLE_TEST, 1);
+    client.removeConstraint(creds, TABLE_TEST, 2);
     client.updateAndFlush(creds, TABLE_TEST, mutation("row1", "cf", "cq", "x"));
     String scanner = client.createScanner(creds, TABLE_TEST, null);
     ScanResult more = client.nextK(scanner, 2);

Modified: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java?rev=1478404&r1=1478403&r2=1478404&view=diff
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/ShellServerTest.java Thu May
 2 15:12:30 2013
@@ -111,7 +111,7 @@ public class ShellServerTest {
   
   static void assertGoodExit(String s, boolean stringPresent) {
     Shell.log.debug(output.get());
-    assertEquals(shell.getExitCode(), 0);
+    assertEquals(0, shell.getExitCode());
 
   if (s.length() > 0)
   assertEquals(s + " present in " + output.get() + " was not " + stringPresent, stringPresent,
output.get().contains(s));
@@ -170,7 +170,7 @@ public class ShellServerTest {
     exec("importtable t2 " + import_, true);
     exec("config -t t2 -np", true, "345M", true);
     exec("getsplits -t t2", true, "row5", true);
-    exec("constraint --list -t t2", true, "VisibilityConstraint=1", true);
+    exec("constraint --list -t t2", true, "VisibilityConstraint=2", true);
     exec("onlinetable t", true);
     exec("deletetable -f t", true);
     exec("deletetable -f t2", true);
@@ -404,7 +404,7 @@ public class ShellServerTest {
     exec("scan", true, "value", true);
     exec("clonetable orig clone");
     // verify constraint, config, and splits were cloned
-    exec("constraint --list -t clone", true, "VisibilityConstraint=1", true);
+    exec("constraint --list -t clone", true, "VisibilityConstraint=2", true);
     exec("config -t clone -np", true, "123M", true);
     exec("getsplits -t clone", true, "a\nb\nc\n");
     // compact
@@ -444,9 +444,9 @@ public class ShellServerTest {
     // constraint
     exec("constraint -l -t !METADATA", true, "MetadataConstraints=1", true);
     exec("createtable c -evc");
-    exec("constraint -l -t c", true, "VisibilityConstraint=1", true);
-    exec("constraint -t c -d 1", true, "Removed constraint 1 from table c");
-    exec("constraint -l -t c", true, "VisibilityConstraint=1", false);
+    exec("constraint -l -t c", true, "VisibilityConstraint=2", true);
+    exec("constraint -t c -d 2", true, "Removed constraint 2 from table c");
+    exec("constraint -l -t c", true, "VisibilityConstraint=2", false);
     exec("deletetable -f c");
   }
   

Modified: accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java?rev=1478404&r1=1478403&r2=1478404&view=diff
==============================================================================
--- accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java (original)
+++ accumulo/trunk/test/src/test/java/org/apache/accumulo/test/TableOperationsIT.java Thu
May  2 15:12:30 2013
@@ -17,11 +17,14 @@
 package org.apache.accumulo.test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -31,6 +34,8 @@ import org.apache.accumulo.core.client.T
 import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.client.admin.DiskUsage;
 import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.constraints.DefaultKeySizeConstraint;
 import org.apache.accumulo.core.security.TablePermission;
 import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
 import org.apache.thrift.TException;
@@ -69,13 +74,13 @@ public class TableOperationsIT {
     assertEquals(1, diskUsage.size());
     assertEquals(0, (long) diskUsage.get(0).getUsage());
     assertEquals("table1", diskUsage.get(0).getTables().iterator().next());
-
+    
     connector.securityOperations().revokeTablePermission(ROOT, "table1", TablePermission.READ);
     try {
       connector.tableOperations().getDiskUsage(Collections.singleton("table1"));
       fail("Should throw securityexception");
-    } catch(AccumuloSecurityException e) {}
-
+    } catch (AccumuloSecurityException e) {}
+    
     connector.tableOperations().delete("table1");
     try {
       connector.tableOperations().getDiskUsage(Collections.singleton("table1"));
@@ -83,6 +88,23 @@ public class TableOperationsIT {
     } catch (TableNotFoundException e) {}
   }
   
+  @Test
+  public void createTable() throws TableExistsException, AccumuloException, AccumuloSecurityException,
TableNotFoundException {
+    connector.tableOperations().create("table1");
+    Iterable<Map.Entry<String,String>> itrProps = connector.tableOperations().getProperties("table1");
+    Map<String,String> props = propsToMap(itrProps);
+    assertEquals(DefaultKeySizeConstraint.class.getName(), props.get(Property.TABLE_CONSTRAINT_PREFIX.toString()
+ "1"));
+    connector.tableOperations().delete("table1");
+  }
+
+  private Map<String,String> propsToMap(Iterable<Map.Entry<String,String>>
props) {
+    Map<String,String> map = new HashMap<String,String>();
+    for (Map.Entry<String,String> prop : props) {
+      map.put(prop.getKey(), prop.getValue());
+    }
+    return map;
+  }
+  
   @AfterClass
   public static void shutDown() throws IOException, InterruptedException {
     accumuloCluster.stop();



Mime
View raw message