hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject hbase git commit: HBASE-14501 NPE in replication with TDE
Date Tue, 13 Oct 2015 03:59:14 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 b26eb0d85 -> 55c2f1325


HBASE-14501 NPE in replication with TDE

Conflicts:
	hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
	hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java

Conflicts:
	hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
	hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55c2f132
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55c2f132
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55c2f132

Branch: refs/heads/branch-1.1
Commit: 55c2f132550ed1ccc4904211ac61fc6035367b24
Parents: b26eb0d
Author: Enis Soztutar <enis@apache.org>
Authored: Mon Oct 12 20:37:34 2015 -0700
Committer: Enis Soztutar <enis@apache.org>
Committed: Mon Oct 12 20:54:01 2015 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hbase/KeyValue.java  |  8 +++--
 .../org/apache/hadoop/hbase/KeyValueUtil.java   |  5 +--
 .../apache/hadoop/hbase/codec/BaseDecoder.java  | 38 +++++++++++++++-----
 .../apache/hadoop/hbase/codec/CellCodec.java    |  1 +
 .../regionserver/wal/SecureWALCellCodec.java    |  9 ++---
 .../regionserver/ReplicationSource.java         |  1 +
 6 files changed, 41 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/55c2f132/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 1275952..ced20af 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hbase.util.Bytes.len;
 
 import java.io.DataInput;
 import java.io.DataOutput;
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -2495,8 +2496,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId,
    * Create a KeyValue reading from the raw InputStream.
    * Named <code>iscreate</code> so doesn't clash with {@link #create(DataInput)}
    * @param in
-   * @return Created KeyValue OR if we find a length of zero, we will return null which
-   * can be useful marking a stream as done.
+   * @return Created KeyValue or throws an exception
    * @throws IOException
    */
   public static KeyValue iscreate(final InputStream in) throws IOException {
@@ -2505,7 +2505,9 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId,
     while (bytesRead < intBytes.length) {
       int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead);
       if (n < 0) {
-        if (bytesRead == 0) return null; // EOF at start is ok
+        if (bytesRead == 0) {
+          throw new EOFException();
+        }
         throw new IOException("Failed read of int, read " + bytesRead + " bytes");
       }
       bytesRead += n;

http://git-wip-us.apache.org/repos/asf/hbase/blob/55c2f132/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
index 20f059d..96bd2e1 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
@@ -184,7 +184,7 @@ public class KeyValueUtil {
    * position to the start of the next KeyValue. Does not allocate a new array or copy data.
    * @param bb
    * @param includesMvccVersion
-   * @param includesTags 
+   * @param includesTags
    */
   public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
       boolean includesTags) {
@@ -248,7 +248,7 @@ public class KeyValueUtil {
     return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
       CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
   }
-  
+
 
   /**
    * Create a KeyValue for the specified row, family and qualifier that would be
@@ -543,6 +543,7 @@ public class KeyValueUtil {
   @Deprecated
   public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
     List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>()
{
+      @Override
       public KeyValue apply(Cell arg0) {
         return KeyValueUtil.ensureKeyValue(arg0);
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/55c2f132/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java
index 3776c08..83549a8 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/BaseDecoder.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.hbase.codec;
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PushbackInputStream;
+
+import javax.annotation.Nonnull;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -32,27 +35,41 @@ import org.apache.hadoop.hbase.Cell;
 @InterfaceAudience.Private
 public abstract class BaseDecoder implements Codec.Decoder {
   protected static final Log LOG = LogFactory.getLog(BaseDecoder.class);
-  protected final InputStream in;
-  private boolean hasNext = true;
+
+  protected final PBIS in;
   private Cell current = null;
 
+  protected static class PBIS extends PushbackInputStream {
+    public PBIS(InputStream in, int size) {
+      super(in, size);
+    }
+
+    public void resetBuf(int size) {
+      this.buf = new byte[size];
+      this.pos = size;
+    }
+  }
+
   public BaseDecoder(final InputStream in) {
-    this.in = in;
+    this.in = new PBIS(in, 1);
   }
 
   @Override
   public boolean advance() throws IOException {
-    if (!this.hasNext) return this.hasNext;
-    if (this.in.available() == 0) {
-      this.hasNext = false;
-      return this.hasNext;
+    int firstByte = in.read();
+    if (firstByte == -1) {
+      return false;
+    } else {
+      in.unread(firstByte);
     }
+
     try {
       this.current = parseCell();
     } catch (IOException ioEx) {
+      in.resetBuf(1); // reset the buffer in case the underlying stream is read from upper
layers
       rethrowEofException(ioEx);
     }
-    return this.hasNext;
+    return true;
   }
 
   private void rethrowEofException(IOException ioEx) throws IOException {
@@ -72,9 +89,12 @@ public abstract class BaseDecoder implements Codec.Decoder {
   }
 
   /**
-   * @return extract a Cell
+   * Extract a Cell.
+   * @return a parsed Cell or throws an Exception. EOFException or a generic IOException
maybe
+   * thrown if EOF is reached prematurely. Does not return null.
    * @throws IOException
    */
+  @Nonnull
   protected abstract Cell parseCell() throws IOException;
 
   @Override

http://git-wip-us.apache.org/repos/asf/hbase/blob/55c2f132/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java
index 77cf80a..6691606 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java
@@ -78,6 +78,7 @@ public class CellCodec implements Codec {
       super(in);
     }
 
+    @Override
     protected Cell parseCell() throws IOException {
       byte [] row = readByteArray(this.in);
       byte [] family = readByteArray(in);

http://git-wip-us.apache.org/repos/asf/hbase/blob/55c2f132/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
index 46f3b88..69181e5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SecureWALCellCodec.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.regionserver.wal;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -84,12 +83,8 @@ public class SecureWALCellCodec extends WALCellCodec {
         return super.parseCell();
       }
       int ivLength = 0;
-      try {
-        ivLength = StreamUtils.readRawVarint32(in);
-      } catch (EOFException e) {
-        // EOF at start is OK
-        return null;
-      }
+
+      ivLength = StreamUtils.readRawVarint32(in);
 
       // TODO: An IV length of 0 could signify an unwrapped cell, when the
       // encoder supports that just read the remainder in directly

http://git-wip-us.apache.org/repos/asf/hbase/blob/55c2f132/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 1db6251..d765d7b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -649,6 +649,7 @@ public class ReplicationSource extends Thread
       if (!CellUtil.matchingRow(cells.get(i), lastCell)) {
         distinctRowKeys++;
       }
+      lastCell = cells.get(i);
     }
     return distinctRowKeys;
   }


Mime
View raw message