accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject svn commit: r1342372 - in /accumulo/trunk: server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java test/system/randomwalk/conf/modules/Shard.xml
Date Thu, 24 May 2012 18:49:29 GMT
Author: kturner
Date: Thu May 24 18:49:28 2012
New Revision: 1342372

URL: http://svn.apache.org/viewvc?rev=1342372&view=rev
Log:
ACCUMULO-420 added random walk test for compaction iterators and fixed bug introduced in previous
checkin against ticket

Added:
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java
Modified:
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
    accumulo/trunk/test/system/randomwalk/conf/modules/Shard.xml

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1342372&r1=1342371&r2=1342372&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Thu May 24 18:49:28 2012
@@ -3252,7 +3252,8 @@ public class Tablet {
           span.data("written", "" + mcs.getEntriesWritten());
           majCStats.add(mcs);
           
-          datafileManager.bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
filesToCompact.size() == 0 ? compactionId.getFirst() : null,
+          datafileManager.bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
+              filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst()
: null,
               new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
           
           // when major compaction produces a file w/ zero entries, it will be deleted...
do not want

Added: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java?rev=1342372&view=auto
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java
(added)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/test/randomwalk/shard/CompactFilter.java
Thu May 24 18:49:28 2012
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.test.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.server.test.randomwalk.State;
+import org.apache.accumulo.server.test.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Test deleting documents by using a compaction filter iterator
+ */
+public class CompactFilter extends Test {
+  
+  @Override
+  public void visit(State state, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String docTableName = (String) state.get("docTableName");
+    Random rand = (Random) state.get("rand");
+    
+    String deleteChar = Integer.toHexString(rand.nextInt(16)) + "";
+    String regex = "^[0-9a-f][" + deleteChar + "].*";
+
+    ArrayList<IteratorSetting> documentFilters = new ArrayList<IteratorSetting>();
+    
+    IteratorSetting is = new IteratorSetting(21, "ii", RegExFilter.class);
+    RegExFilter.setRegexs(is, regex, null, null, null, false);
+    RegExFilter.setNegate(is, true);
+    documentFilters.add(is);
+
+    long t1 = System.currentTimeMillis();
+    state.getConnector().tableOperations().compact(docTableName, null, null, documentFilters,
true, true);
+    long t2 = System.currentTimeMillis();
+    long t3 = t2 - t1;
+    
+    ArrayList<IteratorSetting> indexFilters = new ArrayList<IteratorSetting>();
+    
+    is = new IteratorSetting(21, RegExFilter.class);
+    RegExFilter.setRegexs(is, null, null, regex, null, false);
+    RegExFilter.setNegate(is, true);
+    indexFilters.add(is);
+    
+    t1 = System.currentTimeMillis();
+    state.getConnector().tableOperations().compact(indexTableName, null, null, indexFilters,
true, true);
+    t2 = System.currentTimeMillis();
+    
+    log.debug("Filtered documents using compaction iterators " + regex + " " + (t3) + " "
+ (t2 - t1));
+    
+    BatchScanner bscanner = state.getConnector().createBatchScanner(docTableName, new Authorizations(),
10);
+    
+    List<Range> ranges = new ArrayList<Range>();
+    for (int i = 0; i < 16; i++) {
+      ranges.add(Range.prefix(new Text(Integer.toHexString(i) + "" + deleteChar)));
+    }
+    
+    bscanner.setRanges(ranges);
+    Iterator<Entry<Key,Value>> iter = bscanner.iterator();
+    
+    if (iter.hasNext()) {
+      throw new Exception("Saw unexpected document " + iter.next().getKey());
+    }
+
+    bscanner.close();
+  }
+  
+}

Modified: accumulo/trunk/test/system/randomwalk/conf/modules/Shard.xml
URL: http://svn.apache.org/viewvc/accumulo/trunk/test/system/randomwalk/conf/modules/Shard.xml?rev=1342372&r1=1342371&r2=1342372&view=diff
==============================================================================
--- accumulo/trunk/test/system/randomwalk/conf/modules/Shard.xml (original)
+++ accumulo/trunk/test/system/randomwalk/conf/modules/Shard.xml Thu May 24 18:49:28 2012
@@ -94,6 +94,11 @@
   <edge id="dummy.ToAll" weight="1"/>
 </node>
 
+<node id="shard.CompactFilter">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+
 <node id="dummy.ToAll">
   <edge id="shard.Reindex" weight="3"/>
   <edge id="shard.Flush" weight="3"/>
@@ -102,11 +107,12 @@
   <edge id="shard.Split" weight="40"/>
   <edge id="shard.Merge" weight="20"/>
   <edge id="shard.DeleteWord" weight="9"/>
+  <edge id="shard.CompactFilter" weight="9"/>
   <edge id="shard.DeleteSomeDocs" weight="20"/>
   <edge id="shard.BulkInsert" weight="3"/>
   <edge id="shard.Delete" weight="486"/>
-  <edge id="shard.Insert" weight="4697"/>
-  <edge id="shard.Search" weight="4696"/>
+  <edge id="shard.Insert" weight="4693"/>
+  <edge id="shard.Search" weight="4691"/>
 </node>
 
 </module>



Mime
View raw message