accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1463552 - in /accumulo/branches/1.5/server/src: main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
Date Tue, 02 Apr 2013 14:06:18 GMT
Author: ecn
Date: Tue Apr  2 14:06:18 2013
New Revision: 1463552

URL: http://svn.apache.org/r1463552
Log:
ACCUMULO-1044 add a metadata table constraint to require the bulk load transaction to be alive
when writing the bulk-loaded flags

Modified:
    accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
    accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java

Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java?rev=1463552&r1=1463551&r2=1463552&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
(original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
Tue Apr  2 14:06:18 2013
@@ -34,6 +34,7 @@ import org.apache.accumulo.core.zookeepe
 import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.zookeeper.ZooCache;
 import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher.ZooArbitrator;
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.Logger;
 
@@ -72,6 +73,22 @@ public class MetadataConstraints impleme
     return false;
   }
   
+  static private ArrayList<Short> addViolation(ArrayList<Short> lst, int violation)
{
+    if (lst == null)
+      lst = new ArrayList<Short>();
+    lst.add((short)violation);
+    return lst;
+  }
+  
+  static private ArrayList<Short> addIfNotPresent(ArrayList<Short> lst, int intViolation)
{
+    if (lst == null)
+      return addViolation(lst, intViolation);
+    short violation = (short)intViolation;
+    if (!lst.contains(violation))
+      return addViolation(lst, intViolation);
+    return lst;
+  }
+  
   public List<Short> check(Environment env, Mutation mutation) {
     
     ArrayList<Short> violations = null;
@@ -96,44 +113,30 @@ public class MetadataConstraints impleme
         break;
       
       if (!validTableNameChars[0xff & b]) {
-        if (violations == null)
-          violations = new ArrayList<Short>();
-        if (!violations.contains((short) 4))
-          violations.add((short) 4);
+        violations = addIfNotPresent(violations, 4);
       }
     }
     
     if (!containsSemiC) {
       // see if last row char is <
       if (row.length == 0 || row[row.length - 1] != '<') {
-        if (violations == null)
-          violations = new ArrayList<Short>();
-        if (!violations.contains((short) 4))
-          violations.add((short) 4);
+        violations = addIfNotPresent(violations, 4);
       }
     } else {
       if (row.length == 0) {
-        if (violations == null)
-          violations = new ArrayList<Short>();
-        if (!violations.contains((short) 4))
-          violations.add((short) 4);
+        violations = addIfNotPresent(violations, 4);
       }
     }
     
     if (row.length > 0 && row[0] == '!') {
       if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';'))
{
-        if (violations == null)
-          violations = new ArrayList<Short>();
-        if (!violations.contains((short) 4))
-          violations.add((short) 4);
+        violations = addIfNotPresent(violations, 4);
       }
     }
     
     // ensure row is not less than Constants.METADATA_TABLE_ID
     if (new Text(row).compareTo(new Text(Constants.METADATA_TABLE_ID)) < 0) {
-      if (violations == null)
-        violations = new ArrayList<Short>();
-      violations.add((short) 5);
+      violations = addViolation(violations, 5);
     }
     
     for (ColumnUpdate columnUpdate : colUpdates) {
@@ -141,17 +144,13 @@ public class MetadataConstraints impleme
       
       if (columnUpdate.isDeleted()) {
         if (!isValidColumn(columnUpdate)) {
-          if (violations == null)
-            violations = new ArrayList<Short>();
-          violations.add((short) 2);
+          violations = addViolation(violations, 2);
         }
         continue;
       }
       
       if (columnUpdate.getValue().length == 0 && !columnFamily.equals(Constants.METADATA_SCANFILE_COLUMN_FAMILY))
{
-        if (violations == null)
-          violations = new ArrayList<Short>();
-        violations.add((short) 6);
+        violations = addViolation(violations, 6);
       }
       
       if (columnFamily.equals(Constants.METADATA_DATAFILE_COLUMN_FAMILY)) {
@@ -159,26 +158,49 @@ public class MetadataConstraints impleme
           DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
           
           if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
-            if (violations == null)
-              violations = new ArrayList<Short>();
-            violations.add((short) 1);
+            violations = addViolation(violations, 1);
           }
         } catch (NumberFormatException nfe) {
-          if (violations == null)
-            violations = new ArrayList<Short>();
-          violations.add((short) 1);
+          violations = addViolation(violations, 1);
         } catch (ArrayIndexOutOfBoundsException aiooe) {
-          if (violations == null)
-            violations = new ArrayList<Short>();
-          violations.add((short) 1);
+          violations = addViolation(violations, 1);
         }
       } else if (columnFamily.equals(Constants.METADATA_SCANFILE_COLUMN_FAMILY)) {
         
+      } else if (columnFamily.equals(Constants.METADATA_BULKFILE_COLUMN_FAMILY)) {
+        if (!columnUpdate.isDeleted()) {
+          // splits, which also write the time reference, are allowed to write this reference
even when
+          // the transaction is not running because the other half of the tablet is holding
a reference
+          // to the file.
+          boolean isSplitMutation = false;
+          // When a tablet is assigned, it re-writes the metadata.  It should probably only
update the location information, 
+          // but it writes everything.  We allow it to re-write the bulk information if it
is setting the location. 
+          // See ACCUMULO-1230. 
+          boolean isLocationMutation = false;
+          for (ColumnUpdate update : mutation.getUpdates()) {
+            if (new ColumnFQ(update).equals(Constants.METADATA_TIME_COLUMN)) {
+              isSplitMutation = true;
+            }
+            if (update.getColumnFamily().equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY))
{
+              isLocationMutation = true;
+            }
+          }
+          
+          if (!isSplitMutation && !isLocationMutation) {
+            String tidString = new String(columnUpdate.getValue());
+            long tid = Long.parseLong(tidString);
+            try {
+              if (!new ZooArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid))
{
+                violations = addViolation(violations, 8);
+              }
+            } catch (Exception ex) {
+              violations = addViolation(violations, 8);
+            }
+          }
+        }
       } else {
         if (!isValidColumn(columnUpdate)) {
-          if (violations == null)
-            violations = new ArrayList<Short>();
-          violations.add((short) 2);
+          violations = addViolation(violations, 2);
         } else if (new ColumnFQ(columnUpdate).equals(Constants.METADATA_PREV_ROW_COLUMN)
&& columnUpdate.getValue().length > 0
             && (violations == null || !violations.contains((short) 4))) {
           KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
@@ -188,9 +210,7 @@ public class MetadataConstraints impleme
           boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow())
< 0;
           
           if (!prevEndRowLessThanEndRow) {
-            if (violations == null)
-              violations = new ArrayList<Short>();
-            violations.add((short) 3);
+            violations = addViolation(violations, 3);
           }
         } else if (new ColumnFQ(columnUpdate).equals(Constants.METADATA_LOCK_COLUMN)) {
           if (zooCache == null) {
@@ -211,9 +231,7 @@ public class MetadataConstraints impleme
           }
           
           if (!lockHeld) {
-            if (violations == null)
-              violations = new ArrayList<Short>();
-            violations.add((short) 7);
+            violations = addViolation(violations, 7);
           }
         }
         
@@ -221,7 +239,10 @@ public class MetadataConstraints impleme
     }
     
     if (violations != null) {
-      log.debug(" violating metadata mutation : " + mutation);
+      log.debug("violating metadata mutation : " + new String(mutation.getRow()));
+      for (ColumnUpdate update : mutation.getUpdates()) {
+        log.debug(" update: " + new String(update.getColumnFamily()) + ":" + new String(update.getColumnQualifier())
+ " value " + (update.isDeleted() ? "[delete]" : new String(update.getValue())));
+      }
     }
     
     return violations;
@@ -243,6 +264,8 @@ public class MetadataConstraints impleme
         return "Empty values are not allowed for any " + Constants.METADATA_TABLE_NAME +
" column";
       case 7:
         return "Lock not held in zookeeper by writer";
+      case 8:
+        return "Bulk load transaction no longer running";
     }
     return null;
   }

Modified: accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java?rev=1463552&r1=1463551&r2=1463552&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
(original)
+++ accumulo/branches/1.5/server/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
Tue Apr  2 14:06:18 2013
@@ -105,7 +105,14 @@ public class MetadataConstraintsTest {
     assertNotNull(violations);
     assertEquals(1, violations.size());
     assertEquals(Short.valueOf((short) 4), violations.get(0));
+
+    m = new Mutation(new Text("0;foo"));
+    m.put(Constants.METADATA_BULKFILE_COLUMN_FAMILY, new Text("/someFile"), new Value("12345".getBytes()));
+    violations = mc.check(null, m);
     
+    assertNotNull(violations);
+    assertEquals(1, violations.size());
+    assertEquals(Short.valueOf((short)8), violations.get(0));
   }
   
 }



Mime
View raw message