accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [2/2] git commit: ACCUMULO-1696 fixed compaction debug counts for deep copies
Date Fri, 06 Sep 2013 22:37:28 GMT
ACCUMULO-1696 fixed compaction debug counts for deep copies


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/de24f832
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/de24f832
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/de24f832

Branch: refs/heads/master
Commit: de24f8322f1ef2d4817b335ae90b131f0a7b2c1c
Parents: dee8bbb
Author: Keith Turner <kturner@apache.org>
Authored: Fri Sep 6 18:37:09 2013 -0400
Committer: Keith Turner <kturner@apache.org>
Committed: Fri Sep 6 18:37:09 2013 -0400

----------------------------------------------------------------------
 .../accumulo/server/tabletserver/Compactor.java | 20 +++++--
 .../tabletserver/CountingIteratorTest.java      | 63 ++++++++++++++++++++
 2 files changed, 79 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/de24f832/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
index 4df2637..8001cad 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Compactor.java
@@ -73,9 +73,11 @@ import org.apache.log4j.Logger;
 
 public class Compactor implements Callable<CompactionStats> {
   
-  public class CountingIterator extends WrappingIterator {
+  public static class CountingIterator extends WrappingIterator {
     
     private long count;
+    private ArrayList<CountingIterator> deepCopies;
+    private AtomicLong entriesRead;
     
     public CountingIterator deepCopy(IteratorEnvironment env) {
       return new CountingIterator(this, env);
@@ -84,11 +86,16 @@ public class Compactor implements Callable<CompactionStats> {
     private CountingIterator(CountingIterator other, IteratorEnvironment env) {
       setSource(other.getSource().deepCopy(env));
       count = 0;
+      this.deepCopies = other.deepCopies;
+      this.entriesRead = other.entriesRead;
+      deepCopies.add(this);
     }
     
-    public CountingIterator(SortedKeyValueIterator<Key,Value> source) {
+    public CountingIterator(SortedKeyValueIterator<Key,Value> source, AtomicLong entriesRead)
{
+      deepCopies = new ArrayList<Compactor.CountingIterator>();
       this.setSource(source);
       count = 0;
+      this.entriesRead = entriesRead;
     }
     
     @Override
@@ -106,7 +113,12 @@ public class Compactor implements Callable<CompactionStats> {
     }
     
     public long getCount() {
-      return count;
+      long sum = 0;
+      for (CountingIterator dc : deepCopies) {
+        sum += dc.count;
+      }
+      
+      return count + sum;
     }
   }
 
@@ -454,7 +466,7 @@ public class Compactor implements Callable<CompactionStats> {
         iters.add(imm.compactionIterator());
       }
       
-      CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()));
+      CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()),
entriesRead);
       DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
       ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
       

http://git-wip-us.apache.org/repos/asf/accumulo/blob/de24f832/server/src/test/java/org/apache/accumulo/server/tabletserver/CountingIteratorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/accumulo/server/tabletserver/CountingIteratorTest.java
b/server/src/test/java/org/apache/accumulo/server/tabletserver/CountingIteratorTest.java
new file mode 100644
index 0000000..11ee972
--- /dev/null
+++ b/server/src/test/java/org/apache/accumulo/server/tabletserver/CountingIteratorTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.tabletserver;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.SortedMapIterator;
+import org.apache.accumulo.server.tabletserver.Compactor.CountingIterator;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * 
+ */
+public class CountingIteratorTest {
+  @Test
+  public void testDeepCopyCount() throws IOException {
+    TreeMap<Key,Value> tm = new TreeMap<Key,Value>();
+    
+    tm.put(new Key("r1", "cf1", "cq1"), new Value("data1".getBytes()));
+    tm.put(new Key("r2", "cf1", "cq1"), new Value("data2".getBytes()));
+    
+    SortedMapIterator smi = new SortedMapIterator(tm);
+    
+    CountingIterator ci = new CountingIterator(smi, new AtomicLong(0));
+    CountingIterator dc1 = ci.deepCopy(null);
+    CountingIterator dc2 = ci.deepCopy(null);
+    
+    readAll(ci);
+    readAll(dc1);
+    readAll(dc2);
+    
+    Assert.assertEquals(6, ci.getCount());
+  }
+  
+  private void readAll(CountingIterator ci) throws IOException {
+    ci.seek(new Range(), new HashSet<ByteSequence>(), false);
+    while (ci.hasTop()) {
+      ci.next();
+    }
+  }
+}


Mime
View raw message