hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1406268 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/compactionhook/ test/java/org/apache/hadoop/hbase/regionserver/comp...
Date Tue, 06 Nov 2012 18:44:44 GMT
Author: liyin
Date: Tue Nov  6 18:44:43 2012
New Revision: 1406268

URL: http://svn.apache.org/viewvc?rev=1406268&view=rev
Log:
[HBASE-7099] Make a simpler interface used just for compact hooks in addition to the KeyValueAggregator

Author: adela

Summary: Simpler interface which should be used just for transforming values in KVs. The possible
options for a KV would be to be skipped, transform the value, or leave the value untouched.

Test Plan: unit test

Reviewers: kannan, kranganathan, aaiyer, liyintang

Reviewed By: liyintang

CC: hbase-eng@, erling, liujiakai, jfan, ajaysomani

Differential Revision: https://phabricator.fb.com/D611834

Task ID: 1838741, 1749101

Added:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java
      - copied, changed from r1405917, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/HConstants.java Tue Nov 
6 18:44:43 2012
@@ -595,11 +595,15 @@ public final class HConstants {
   public static final int IPC_CALL_PARAMETER_LENGTH_MAX = 1000;
 
   /**
-   * used in Configuration to get/set the KV aggregator for compaction
-   * hooks
+   * Used in Configuration to get/set the KV aggregator
    */
   public static final String KV_AGGREGATOR = "kvaggregator";
 
+  /**
+   * Used in Configuration to get/set the compaction hook
+   */
+  public static final String COMPACTION_HOOK = "compaction_hook";
+
   private HConstants() {
     // Can't be instantiated with this ctor.
   }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Nov  6
18:44:43 2012
@@ -967,6 +967,14 @@ public class KeyValue implements Writabl
   }
 
   /**
+   * @return return True if Put type.
+   */
+  public boolean isPut() {
+    int t = getType();
+    return (t == Type.Put.getCode()) ? true : false;
+  }
+
+  /**
    * @return True if this KV is a {@link KeyValue.Type#Delete} type.
    */
   public boolean isDeleteType() {
@@ -2188,4 +2196,14 @@ public class KeyValue implements Writabl
       final int valueLength) {
     return ROW_OFFSET + keyLength + valueLength;
   }
+
+  /**
+   * Overwrites the value in the KV, keeps the other properties the same
+   */
+  public KeyValue modifyValueAndClone(byte[] newValue) {
+    KeyValue newKV = new KeyValue(this.getRow(), this.getFamily(),
+        this.getQualifier(), this.getTimestamp(), newValue);
+    newKV.setMemstoreTS(this.getMemstoreTS());
+    return newKV;
+  }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Tue Nov  6 18:44:43 2012
@@ -343,7 +343,7 @@ public class HRegion implements HeapSize
     }
     oldVal.addAndGet(amount);
   }
-  
+
   public static void clearNumericPersistentMetric(String key) {
     numericPersistentMetrics.remove(key);
   }
@@ -1175,7 +1175,7 @@ public class HRegion implements HeapSize
    * @return split row if split is needed
    * @throws IOException e
    */
-  byte [] compactStores(final boolean majorCompaction)
+  public byte [] compactStores(final boolean majorCompaction)
   throws IOException {
     if (majorCompaction) {
       this.triggerMajorCompaction();
@@ -1972,7 +1972,7 @@ public class HRegion implements HeapSize
    * @param methodName "multiput_/multidelete_" to update metrics correctly.
    * @throws IOException
    */
-  public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,

+  public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
       String methodName) throws IOException {
     this.writeRequests.incrTotalRequstCount();
     BatchOperationInProgress<Pair<Mutation, Integer>> batchOp =
@@ -1998,7 +1998,7 @@ public class HRegion implements HeapSize
     return batchOp.retCodes;
   }
 
-  private long doMiniBatchOp(BatchOperationInProgress<Pair<Mutation, Integer>>
batchOp, 
+  private long doMiniBatchOp(BatchOperationInProgress<Pair<Mutation, Integer>>
batchOp,
       String methodNameForMetricsUpdate) throws IOException {
     String signature = null;
     // variable to note if all Put items are for the same CF -- metrics related
@@ -2023,7 +2023,7 @@ public class HRegion implements HeapSize
       byte[] previousRow = null;
       Integer previousLockID = null;
       Integer currentLockID = null;
-      
+
       while (lastIndexExclusive < batchOp.operations.length) {
         Pair<Mutation, Integer> nextPair = batchOp.operations[lastIndexExclusive];
         Mutation op = nextPair.getFirst();
@@ -2034,7 +2034,7 @@ public class HRegion implements HeapSize
           checkFamilies(op.getFamilyMap().keySet());
           checkTimestamps(op, now);
         }
-        
+
         if (previousRow == null || !Bytes.equals(previousRow, op.getRow()) ||
             (providedLockId != null && !previousLockID.equals(providedLockId))) {
           // If we haven't got any rows in our batch, we should block to
@@ -2046,18 +2046,18 @@ public class HRegion implements HeapSize
             assert !shouldBlock : "Should never fail to get lock when blocking";
             break; // stop acquiring more rows for this batch
           }
-          
+
           if (providedLockId == null) {
             acquiredLocks.add(currentLockID);
           }
-          
+
           // reset the previous row and lockID with the current one
           previousRow = op.getRow();
           previousLockID = currentLockID;
         }
         lastIndexExclusive++;
         numReadyToWrite++;
-        
+
         // if first time around, designate expected signature for metric
         // else, if all have been consistent so far, check if it still holds
         // all else, designate failure signature and mark as unclear
@@ -2087,7 +2087,7 @@ public class HRegion implements HeapSize
       // ----------------------------------
       for (int i = firstIndex; i < lastIndexExclusive; i++) {
         Mutation op = batchOp.operations[i].getFirst();
-        
+
         if (op instanceof Put) {
           updateKVTimestamps(
               op.getFamilyMap().values(),
@@ -2136,7 +2136,7 @@ public class HRegion implements HeapSize
       if (locked) {
         this.updatesLock.readLock().unlock();
       }
-      
+
       releaseRowLocks(acquiredLocks);
 
       // do after lock
@@ -2195,7 +2195,7 @@ public class HRegion implements HeapSize
       try {
         result = get(get);
         boolean matches = false;
-        
+
         if (result.size() == 0) {
           if (expectedValue == null ) {
             matches = true;
@@ -2394,7 +2394,7 @@ public class HRegion implements HeapSize
   private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap)
{
     return applyFamilyMapToMemstore(familyMap, null);
   }
-  
+
   private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap,
                  MultiVersionConsistencyControl.WriteEntry writeEntryToUse) {
     long start = EnvironmentEdgeManager.currentTimeMillis();
@@ -2896,7 +2896,7 @@ public class HRegion implements HeapSize
       lockedRows.notifyAll();
     }
   }
-  
+
   /**
    * Release the row locks!
    * @param lockidList The list of the lock ID to release.
@@ -2969,7 +2969,7 @@ public class HRegion implements HeapSize
   public void bulkLoadHFile(String hfilePath, byte[] familyName, boolean assignSeqId)
   throws IOException {
     long seqId = this.log.obtainSeqNum();
-    
+
     splitsAndClosesLock.readLock().lock();
     try {
       Store store = getStore(familyName);
@@ -2982,7 +2982,7 @@ public class HRegion implements HeapSize
       splitsAndClosesLock.readLock().unlock();
     }
   }
-  
+
   public void bulkLoadHFile(String hfilePath, byte[] familyName)
   throws IOException {
     bulkLoadHFile(hfilePath, familyName, false);
@@ -3791,7 +3791,7 @@ public class HRegion implements HeapSize
     boolean flush = false;
 
     Integer lid = null;
-    
+
     splitsAndClosesLock.readLock().lock();
     try {
       // 1. run all pre-hooks before the atomic operation
@@ -3859,7 +3859,7 @@ public class HRegion implements HeapSize
       }
       // 10. run all coprocessor post hooks, after region lock is released
       // NOT required in 0.89. coprocessors are not supported.
-      
+
     } finally {
       if (lid != null) {
         // 11. release the row lock

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/RestrictedKeyValue.java
Tue Nov  6 18:44:43 2012
@@ -0,0 +1,76 @@
+
+/*
+ * Copyright The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+/**
+ * <p>
+ * This is a wrapper around the {@link KeyValue} to insure the user is disabled
+ * from modifying other fields in the KeyValue except the value. Please don't
+ * try to modify the visibility of keyValue from private to public, or generate
+ * a setter, you will introduce huge harms.
+ * </p>
+ *
+ *
+ * <p>
+ * We put this class in regionserver package since we want to be able to access
+ * the getKeyValue() method in {@link Store} and we want to restricted custom
+ * Compaction Hooks to access the method.
+ * </p>
+ */
+public class RestrictedKeyValue {
+  private KeyValue keyValue;
+
+  public RestrictedKeyValue(KeyValue keyValue) {
+    this.keyValue = keyValue;
+  }
+
+  /**
+   * If you don't call this method, the KV will stay unchanged.
+   * @param newValue
+   */
+  public void modifyValue(byte[] newValue) {
+    this.keyValue = keyValue.modifyValueAndClone(newValue);
+  }
+
+  public byte[] getValue() {
+    return keyValue.getValue();
+  }
+
+  /** don't change the modifier */
+  KeyValue getKeyValue() {
+    return keyValue;
+  }
+
+  public byte[] getRow() {
+    return keyValue.getRow();
+  }
+
+  public byte[] getFamily() {
+    return keyValue.getFamily();
+  }
+
+  public byte[] getQualifier() {
+    return keyValue.getQualifier();
+  }
+
+  public long getTimestamp() {
+    return keyValue.getTimestamp();
+  }
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue
Nov  6 18:44:43 2012
@@ -61,6 +61,7 @@ import org.apache.hadoop.hbase.io.hfile.
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.regionserver.compactionhook.CompactionHook;
 import org.apache.hadoop.hbase.regionserver.kvaggregator.DefaultKeyValueAggregator;
 import org.apache.hadoop.hbase.regionserver.kvaggregator.KeyValueAggregator;
 import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
@@ -147,6 +148,7 @@ public class Store extends SchemaConfigu
   final KeyValue.KVComparator comparator;
 
   private Class<KeyValueAggregator> aggregatorClass = null;
+  private CompactionHook compactHook = null;
 
   /**
    * Constructor
@@ -251,6 +253,18 @@ public class Store extends SchemaConfigu
         throw new IllegalArgumentException(e);
       }
     }
+    String compactHookString = conf.get(HConstants.COMPACTION_HOOK);
+    if (compactHookString != null && !compactHookString.isEmpty()) {
+      try {
+        this.compactHook = (CompactionHook) Class.forName(compactHookString).newInstance();
+      } catch (InstantiationException e) {
+        throw new IllegalArgumentException(e);
+      } catch (IllegalAccessException e) {
+        throw new IllegalArgumentException(e);
+      } catch (ClassNotFoundException e) {
+        throw new IllegalArgumentException(e);
+      }
+    }
   }
 
   /**
@@ -435,7 +449,7 @@ public class Store extends SchemaConfigu
   }
 
   /**
-   * Adds a value to the memstore
+   * Deletes a value to the memstore
    *
    * @param kv
    * @return memstore size delta
@@ -1187,8 +1201,15 @@ public class Store extends SchemaConfigu
               if (kv.getMemstoreTS() <= smallestReadPoint) {
                 kv.setMemstoreTS(0);
               }
-              writer.append(kv);
-
+              if (compactHook != null && kv.isPut()) {
+                RestrictedKeyValue restrictedKv = new RestrictedKeyValue(kv);
+                RestrictedKeyValue modifiedKv = compactHook.transform(restrictedKv);
+                if (modifiedKv != null) {
+                  writer.append(modifiedKv.getKeyValue());
+                }
+              } else {
+                writer.append(kv);
+              }
               // check periodically to see if a system stop is requested
               if (Store.closeCheckInterval > 0) {
                 bytesWritten += kv.getLength();
@@ -1833,7 +1854,7 @@ public class Store extends SchemaConfigu
 
   public static final long FIXED_OVERHEAD =
       ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
-          + (18 * ClassSize.REFERENCE) + (5 * Bytes.SIZEOF_LONG)
+          + (19 * ClassSize.REFERENCE) + (5 * Bytes.SIZEOF_LONG)
           + (4 * Bytes.SIZEOF_INT) + 2 * Bytes.SIZEOF_BOOLEAN) ;
 
   public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/compactionhook/CompactionHook.java
Tue Nov  6 18:44:43 2012
@@ -0,0 +1,46 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.hbase.regionserver.compactionhook;
+
+import org.apache.hadoop.hbase.regionserver.RestrictedKeyValue;
+
+/**
+ * <p>
+ * This interface should be implemented for compaction hooks. The implementation
+ * should be stateless, because a common instance of the implementation might be
+ * reused across multiple active compactions.To see an example of a compaction
+ * hook implementation go here: {@link LowerToUpperCompactionHook}.
+ * </p>
+ */
+
+public interface CompactionHook {
+
+  /**
+   * This should be called to transform a KeyValue's value into a desired
+   * format. You can do the following transformations:
+   * <ul>
+   * <li>return null: that way you will skip this KV </li>
+   * <li>return the same kv: if you don't want to apply any changes to the KV</li>
+   * <li> transform only the value of the KV into the desired format</li>
+   * </ul>
+   *
+   * @param kv - the KeyValueForCompactionHooks that we want to transform
+   * @return the resulting KeyValueForCompactionHooks
+   */
+  RestrictedKeyValue transform(RestrictedKeyValue kv);
+
+}

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java
(added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/LowerToUpperCompactionHook.java
Tue Nov  6 18:44:43 2012
@@ -0,0 +1,44 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hbase.regionserver.compactionhook;
+
+import org.apache.hadoop.hbase.regionserver.RestrictedKeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * <p>
+ * Dummy compaction hook which transforms a lower-case value in KV to
+ * upper-case, implements CompactionHook interface.
+ * </p>
+ *
+ */
+public class LowerToUpperCompactionHook implements CompactionHook {
+
+  @Override
+  public RestrictedKeyValue transform (RestrictedKeyValue kv) {
+    byte[] newValue;
+    String currentValue = Bytes.toString(kv.getValue());
+    /**
+     * transform it to uppercase.
+     */
+    String newValueString = currentValue.toUpperCase();
+    newValue = Bytes.toBytes(newValueString);
+    kv.modifyValue(newValue);
+    return kv;
+  }
+}

Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java?rev=1406268&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java
(added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/SkipCompactionHook.java
Tue Nov  6 18:44:43 2012
@@ -0,0 +1,37 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hbase.regionserver.compactionhook;
+
+import org.apache.hadoop.hbase.regionserver.RestrictedKeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class SkipCompactionHook implements CompactionHook {
+
+  /**
+   * Returns null only if the value of the KV contains 'c' or 'C' in its String
+   * representation
+   */
+  @Override
+  public RestrictedKeyValue transform(RestrictedKeyValue kv) {
+    String currentValue = Bytes.toString(kv.getValue());
+    if (currentValue.contains("C") || currentValue.contains("c")) {
+      return null;
+    }
+    return kv;
+  }
+}

Copied: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java
(from r1405917, hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java)
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java?p2=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java&p1=hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java&r1=1405917&r2=1406268&rev=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/compactionhook/TestCompactionHook.java
Tue Nov  6 18:44:43 2012
@@ -1,4 +1,20 @@
-package org.apache.hadoop.hbase.regionserver.kvaggregator;
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.hadoop.hbase.regionserver.compactionhook;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -21,8 +37,9 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 @SuppressWarnings("deprecation")
-public class TestKeyValueAggregator {
-  private static byte[] TABLE = Bytes.toBytes("TestKeyValueAggregator");
+public class TestCompactionHook {
+
+  private static byte[] TABLE = Bytes.toBytes("TestCompactionHook");
   private static byte[] FAMILY = Bytes.toBytes("family");
   private static byte[] START_KEY = Bytes.toBytes("aaa");
   private static byte[] END_KEY = Bytes.toBytes("zzz");
@@ -42,12 +59,13 @@ public class TestKeyValueAggregator {
         .setCompressionType(Compression.Algorithm.NONE));
     TEST_UTIL
         .getConfiguration()
-        .set(HConstants.KV_AGGREGATOR,
-            "org.apache.hadoop.hbase.regionserver.kvaggregator.LowerToUpperAggregator");
+        .set(
+            HConstants.COMPACTION_HOOK,
+            "org.apache.hadoop.hbase.regionserver.compactionhook.LowerToUpperCompactionHook");
   }
 
   @Test
-  public void testDummyKvAggregator() throws Exception {
+  public void testLowerToUpperHook() throws Exception {
     HRegion r = HBaseTestCase.createNewHRegion(TESTTABLEDESC, START_KEY,
         END_KEY, TEST_UTIL.getConfiguration());
     Put[] puts = new Put[25];
@@ -61,13 +79,19 @@ public class TestKeyValueAggregator {
       puts[i] = put;
     }
     r.put(puts);
+    // without calling this, the compaction will not happen
+    r.flushcache();
+    r.compactStores(true);
+    //
     Scan scan = new Scan();
     InternalScanner s = r.getScanner(scan);
     List<KeyValue> results = new ArrayList<KeyValue>();
     while (s.next(results))
       ;
-    // check if the values in results are in upper case
     s.close();
+    // check if the values in results are in upper case
+    // this should return true, we are flushing the HRegion, so we expect the
+    // compaction to be done
     Assert.assertTrue(checkIfLowerCase(results));
     // check if we got all 25 results back
     Assert.assertEquals(25, results.size());
@@ -83,4 +107,39 @@ public class TestKeyValueAggregator {
     }
     return true;
   }
+
+  @Test
+  public void testSkipHook () throws Exception {
+    TEST_UTIL
+    .getConfiguration()
+    .set(
+        HConstants.COMPACTION_HOOK,
+        "org.apache.hadoop.hbase.regionserver.compactionhook.SkipCompactionHook");
+    HRegion r = HBaseTestCase.createNewHRegion(TESTTABLEDESC, START_KEY,
+        END_KEY, TEST_UTIL.getConfiguration());
+    Put[] puts = new Put[25];
+    // put some strings
+    for (int i = 0; i < 25; i++) {
+      byte[] row = Bytes.toBytes("row" + i);
+      Put put = new Put(row);
+      byte[] qualifier = Bytes.toBytes("qual" + i);
+      byte[] value = Bytes.toBytes("ab" + (char) (i + 97));
+      put.add(FAMILY, qualifier, value);
+      puts[i] = put;
+    }
+    r.put(puts);
+    // without calling this, the compaction will not happen
+    r.flushcache();
+    r.compactStores(true);
+    //
+    Scan scan = new Scan();
+    InternalScanner s = r.getScanner(scan);
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    while (s.next(results))
+      ;
+    s.close();
+    // check if we got 24 results back, since "abc" should be skipped.
+    Assert.assertEquals(24, results.size());
+  }
+
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/LowerToUpperAggregator.java
Tue Nov  6 18:44:43 2012
@@ -1,7 +1,25 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
 package org.apache.hadoop.hbase.regionserver.kvaggregator;
 
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
+import org.apache.hadoop.hbase.util.Bytes;
 
 /**
  * <p>
@@ -18,12 +36,12 @@ public class LowerToUpperAggregator impl
   @Override
   public KeyValue process(KeyValue kv) {
     byte[] newValue;
-    String currentValue = new String(kv.getValue());
+    String currentValue = Bytes.toString(kv.getValue());
     /**
      * transform it to uppercase.
      */
     String newValueString = currentValue.toUpperCase();
-    newValue = newValueString.getBytes();
+    newValue = Bytes.toBytes(newValueString);
     KeyValue newKv = new KeyValue(kv.getRow(), kv.getFamily(),
         kv.getQualifier(), kv.getTimestamp(), newValue);
     return newKv;

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java?rev=1406268&r1=1406267&r2=1406268&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/kvaggregator/TestKeyValueAggregator.java
Tue Nov  6 18:44:43 2012
@@ -1,3 +1,20 @@
+/**
+ * Copyright The Apache Software Foundation.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
 package org.apache.hadoop.hbase.regionserver.kvaggregator;
 
 import java.util.ArrayList;



Mime
View raw message