Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 41E69110D1 for ; Wed, 27 Aug 2014 16:19:01 +0000 (UTC) Received: (qmail 26781 invoked by uid 500); 27 Aug 2014 16:19:01 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 26743 invoked by uid 500); 27 Aug 2014 16:19:01 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 26734 invoked by uid 99); 27 Aug 2014 16:19:00 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 27 Aug 2014 16:19:00 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B6C399CBBE9; Wed, 27 Aug 2014 16:19:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jxiang@apache.org To: commits@hbase.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: git commit: HBASE-11788 HBase is not deleting the cell when a Put with a KeyValue, KeyValue.Type.Delete is submitted Date: Wed, 27 Aug 2014 16:19:00 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/branch-1 9bd9b26a0 -> 095a0e9aa HBASE-11788 HBase is not deleting the cell when a Put with a KeyValue, KeyValue.Type.Delete is submitted Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/095a0e9a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/095a0e9a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/095a0e9a Branch: refs/heads/branch-1 Commit: 095a0e9aa2e8d007749de92bdfa8a2932ef81c2b Parents: 9bd9b26 Author: Srikanth Srungarapu Authored: Tue Aug 26 11:25:24 2014 -0700 Committer: Jimmy Xiang Committed: Wed Aug 27 09:16:14 2014 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/protobuf/ProtobufUtil.java | 98 +++++++++++++------- .../hadoop/hbase/client/TestPutWithDelete.java | 96 +++++++++++++++++++ 2 files changed, 162 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/095a0e9a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index aaf9bce..df6016a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -18,24 +18,17 @@ package org.apache.hadoop.hbase.protobuf; -import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.NavigableSet; - +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.Parser; +import com.google.protobuf.RpcChannel; +import com.google.protobuf.Service; +import com.google.protobuf.ServiceException; +import com.google.protobuf.TextFormat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; @@ -138,17 +131,23 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.token.Token; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Lists; -import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.Parser; -import com.google.protobuf.RpcChannel; -import com.google.protobuf.Service; -import com.google.protobuf.ServiceException; -import com.google.protobuf.TextFormat; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableSet; + +import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME; /** * Protobufs utility. @@ -579,9 +578,21 @@ public final class ProtobufUtil { for(int i = 0; i< array.length; i++) { tagArray[i] = (Tag)array[i]; } - put.addImmutable(family, qualifier, ts, value, tagArray); + if(qv.hasDeleteType()) { + byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null; + put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts, + fromDeleteType(qv.getDeleteType()), null, tags)); + } else { + put.addImmutable(family, qualifier, ts, value, tagArray); + } } else { - put.addImmutable(family, qualifier, ts, value); + if(qv.hasDeleteType()) { + byte[] qual = qv.hasQualifier() ? qv.getQualifier().toByteArray() : null; + put.add(new KeyValue(proto.getRow().toByteArray(), family, qual, ts, + fromDeleteType(qv.getDeleteType()))); + } else{ + put.addImmutable(family, qualifier, ts, value); + } } } } @@ -1167,7 +1178,7 @@ public final class ProtobufUtil { valueBuilder.setTags(ByteStringer.wrap(kv.getTagsArray(), kv.getTagsOffset(), kv.getTagsLength())); } - if (type == MutationType.DELETE) { + if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(kv))) { KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType()); valueBuilder.setDeleteType(toDeleteType(keyValueType)); } @@ -1470,6 +1481,29 @@ public final class ProtobufUtil { } /** + * Convert a protocol buffer DeleteType to delete KeyValue type. + * + * @param protocol buffer DeleteType + * @return type + * @throws IOException + */ + public static KeyValue.Type fromDeleteType( + DeleteType type) throws IOException { + switch (type) { + case DELETE_ONE_VERSION: + return KeyValue.Type.Delete; + case DELETE_MULTIPLE_VERSIONS: + return KeyValue.Type.DeleteColumn; + case DELETE_FAMILY: + return KeyValue.Type.DeleteFamily; + case DELETE_FAMILY_VERSION: + return KeyValue.Type.DeleteFamilyVersion; + default: + throw new IOException("Unknown delete type: " + type); + } + } + + /** * Convert a stringified protocol buffer exception Parameter to a Java Exception * * @param parameter the protocol buffer Parameter to convert http://git-wip-us.apache.org/repos/asf/hbase/blob/095a0e9a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestPutWithDelete.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestPutWithDelete.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestPutWithDelete.java new file mode 100644 index 0000000..028a8ba --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestPutWithDelete.java @@ -0,0 +1,96 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertTrue; + +@Category(MediumTests.class) +public class TestPutWithDelete { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testHbasePutDeleteCell() throws Exception { + final TableName tableName = TableName.valueOf("TestPutWithDelete"); + final byte[] rowKey = Bytes.toBytes("12345"); + final byte[] family = Bytes.toBytes("cf"); + HTable table = TEST_UTIL.createTable(tableName, family); + TEST_UTIL.waitTableAvailable(tableName.getName(), 5000); + try { + // put one row + Put put = new Put(rowKey); + put.add(family, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.add(family, Bytes.toBytes("B"), Bytes.toBytes("b")); + put.add(family, Bytes.toBytes("C"), Bytes.toBytes("c")); + table.put(put); + // get row back and assert the values + Get get = new Get(rowKey); + Result result = table.get(get); + assertTrue("Column A value should be a", + Bytes.toString(result.getValue(family, Bytes.toBytes("A"))).equals("a")); + assertTrue("Column B value should be b", + Bytes.toString(result.getValue(family, Bytes.toBytes("B"))).equals("b")); + assertTrue("Column C value should be c", + Bytes.toString(result.getValue(family, Bytes.toBytes("C"))).equals("c")); + // put the same row again with C column deleted + put = new Put(rowKey); + put.add(family, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.add(family, Bytes.toBytes("B"), Bytes.toBytes("b")); + KeyValue marker = new KeyValue(rowKey, family, Bytes.toBytes("C"), + HConstants.LATEST_TIMESTAMP, KeyValue.Type.DeleteColumn); + put.add(marker); + table.put(put); + // get row back and assert the values + get = new Get(rowKey); + result = table.get(get); + assertTrue("Column A value should be a", + Bytes.toString(result.getValue(family, Bytes.toBytes("A"))).equals("a")); + assertTrue("Column B value should be b", + Bytes.toString(result.getValue(family, Bytes.toBytes("B"))).equals("b")); + assertTrue("Column C should not exist", + result.getValue(family, Bytes.toBytes("C")) == null); + } finally { + table.close(); + } + } +} + +