phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rajeshb...@apache.org
Subject phoenix git commit: PHOENIX-4440 Local index split/merge IT tests are failing(Rajeshbabu)
Date Thu, 22 Mar 2018 08:53:51 GMT
Repository: phoenix
Updated Branches:
  refs/heads/5.x-HBase-2.0 8ee15d9f8 -> 31c674a82


PHOENIX-4440 Local index split/merge IT tests are failing(Rajeshbabu)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/31c674a8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/31c674a8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/31c674a8

Branch: refs/heads/5.x-HBase-2.0
Commit: 31c674a826b188c93713d6057ae9dc3238d564c3
Parents: 8ee15d9
Author: Rajeshbabu Chintaguntla <rajeshbabu@apache.org>
Authored: Thu Mar 22 14:23:31 2018 +0530
Committer: Rajeshbabu Chintaguntla <rajeshbabu@apache.org>
Committed: Thu Mar 22 14:23:31 2018 +0530

----------------------------------------------------------------------
 .../phoenix/end2end/LocalIndexSplitMergeIT.java | 60 +++++++++++++++++++-
 .../index/MutableIndexSplitForwardScanIT.java   |  3 +-
 .../index/MutableIndexSplitReverseScanIT.java   |  3 +-
 .../hadoop/hbase/client/RegionInfoUtil.java     | 24 ++++++++
 .../regionserver/IndexHalfStoreFileReader.java  | 45 ++++++++++++---
 .../IndexHalfStoreFileReaderGenerator.java      | 19 +++++--
 .../LocalIndexStoreFileScanner.java             | 43 +++++++-------
 .../coprocessor/BaseScannerRegionObserver.java  |  2 +-
 .../query/ConnectionQueryServicesImpl.java      | 12 ++--
 .../java/org/apache/phoenix/util/ScanUtil.java  |  7 +--
 .../phoenix/execute/CorrelatePlanTest.java      |  1 -
 .../index/write/TestWALRecoveryCaching.java     |  9 ++-
 12 files changed, 174 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java
index c27acee..d6783ff 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LocalIndexSplitMergeIT.java
@@ -52,7 +52,6 @@ import org.junit.experimental.categories.Category;
 import com.google.common.collect.Maps;
 
 @Category(NeedsOwnMiniClusterTest.class)
-@Ignore
 public class LocalIndexSplitMergeIT extends BaseTest {
 
     @BeforeClass
@@ -265,4 +264,63 @@ public class LocalIndexSplitMergeIT extends BaseTest {
         }
     }
 
+    @Test
+    public void testLocalIndexScanWithMergeSpecialCase() throws Exception {
+        String schemaName = generateUniqueName();
+        String tableName = schemaName + "." + generateUniqueName();
+        String indexName = "IDX_" + generateUniqueName();
+        TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(),
false);
+        createBaseTable(tableName, "('a','aaaab','def')");
+        Connection conn1 = getConnectionForLocalIndexTest();
+        try {
+            String[] strings =
+                    { "aa", "aaa", "aaaa", "bb", "cc", "dd", "dff", "g", "h", "i", "j", "k",
"l",
+                            "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y",
"z" };
+            for (int i = 0; i < 26; i++) {
+                conn1.createStatement()
+                        .execute("UPSERT INTO " + tableName + " values('" + strings[i] +
"'," + i
+                                + "," + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i]
+ "')");
+            }
+            conn1.commit();
+            conn1.createStatement()
+                    .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
+            conn1.createStatement()
+            .execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName + "(k3)");
+
+            Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
+            List<RegionInfo> regionsOfUserTable =
+                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
+                        false);
+            admin.mergeRegionsAsync(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
+                regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
+            regionsOfUserTable =
+                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
+                        false);
+
+            while (regionsOfUserTable.size() != 3) {
+                Thread.sleep(100);
+                regionsOfUserTable =
+                        MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
+                            false);
+            }
+            String query = "SELECT t_id,k1,v1 FROM " + tableName;
+            ResultSet rs = conn1.createStatement().executeQuery(query);
+            for (int j = 0; j < 26; j++) {
+                assertTrue(rs.next());
+                assertEquals(strings[25-j], rs.getString("t_id"));
+                assertEquals(25-j, rs.getInt("k1"));
+                assertEquals(strings[j], rs.getString("V1"));
+            }
+            query = "SELECT t_id,k1,k3 FROM " + tableName;
+            rs = conn1.createStatement().executeQuery(query);
+            for (int j = 0; j < 26; j++) {
+                assertTrue(rs.next());
+                assertEquals(strings[j], rs.getString("t_id"));
+                assertEquals(j, rs.getInt("k1"));
+                assertEquals(j + 2, rs.getInt("k3"));
+            }
+        } finally {
+            conn1.close();
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java
index 83320c2..75e2ae2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitForwardScanIT.java
@@ -19,8 +19,7 @@ package org.apache.phoenix.end2end.index;
 
 import org.junit.Test;
 
-//TODO: re-enable once PHOENIX-4273 is fixed
-public abstract class MutableIndexSplitForwardScanIT extends MutableIndexSplitIT {
+public class MutableIndexSplitForwardScanIT extends MutableIndexSplitIT {
 
     public MutableIndexSplitForwardScanIT(boolean localIndex, boolean multiTenant) {
         super(localIndex, multiTenant);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java
index f57dfdd..d7afb31 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexSplitReverseScanIT.java
@@ -19,8 +19,7 @@ package org.apache.phoenix.end2end.index;
 
 import org.junit.Test;
 
-//TODO: re-enable once PHOENIX-4273 is fixeds
-public abstract class MutableIndexSplitReverseScanIT extends MutableIndexSplitIT {
+public class MutableIndexSplitReverseScanIT extends MutableIndexSplitIT {
 
     public MutableIndexSplitReverseScanIT(boolean localIndex, boolean multiTenant) {
         super(localIndex, multiTenant);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/main/java/org/apache/hadoop/hbase/client/RegionInfoUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/client/RegionInfoUtil.java
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/client/RegionInfoUtil.java
new file mode 100644
index 0000000..fd83af7
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/client/RegionInfoUtil.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+  
+public class RegionInfoUtil {
+    public static byte[] toByteArray(RegionInfo regionInfo) {
+        return RegionInfo.toByteArray(regionInfo);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
index 8e590f6..107e9ca 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_START_ROW_SUFFIX;
+
 import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -25,13 +27,14 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.index.IndexMaintainer;
 
 /**
@@ -58,8 +61,10 @@ public class IndexHalfStoreFileReader extends StoreFileReader {
     private final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers;
     private final byte[][] viewConstants;
     private final int offset;
-    private final RegionInfo regionInfo;
+    private final RegionInfo childRegionInfo;
     private final byte[] regionStartKeyInHFile;
+    private final AtomicInteger refCount;
+    private final RegionInfo currentRegion;
 
     /**
      * @param fs
@@ -81,18 +86,21 @@ public class IndexHalfStoreFileReader extends StoreFileReader {
             final Configuration conf,
             final Map<ImmutableBytesWritable, IndexMaintainer> indexMaintainers,
             final byte[][] viewConstants, final RegionInfo regionInfo,
-            byte[] regionStartKeyInHFile, byte[] splitKey, boolean primaryReplicaStoreFile)
throws IOException {
-        super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, new AtomicInteger(0),
false,
+            byte[] regionStartKeyInHFile, byte[] splitKey, boolean primaryReplicaStoreFile,
+            AtomicInteger refCount, RegionInfo currentRegion) throws IOException {
+        super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, refCount, false,
                 conf);
         this.splitkey = splitKey == null ? r.getSplitKey() : splitKey;
         // Is it top or bottom half?
         this.top = Reference.isTopFileRegion(r.getFileRegion());
-        this.splitRow = CellUtil.cloneRow(KeyValueUtil.createKeyValueFromKey(splitkey));
+        this.splitRow = CellUtil.cloneRow(new KeyValue.KeyOnlyKeyValue(splitkey));
         this.indexMaintainers = indexMaintainers;
         this.viewConstants = viewConstants;
-        this.regionInfo = regionInfo;
+        this.childRegionInfo = regionInfo;
         this.regionStartKeyInHFile = regionStartKeyInHFile;
         this.offset = regionStartKeyInHFile.length;
+        this.refCount = refCount;
+        this.currentRegion = currentRegion;
     }
 
     public int getOffset() {
@@ -108,7 +116,7 @@ public class IndexHalfStoreFileReader extends StoreFileReader {
     }
 
     public RegionInfo getRegionInfo() {
-        return regionInfo;
+        return childRegionInfo;
     }
 
     public byte[] getRegionStartKeyInHFile() {
@@ -131,7 +139,28 @@ public class IndexHalfStoreFileReader extends StoreFileReader {
     public StoreFileScanner getStoreFileScanner(boolean cacheBlocks, boolean pread,
             boolean isCompaction, long readPt, long scannerOrder,
             boolean canOptimizeForNonNullColumn) {
+        refCount.incrementAndGet();
         return new LocalIndexStoreFileScanner(this, cacheBlocks, pread, isCompaction, readPt,
                 scannerOrder, canOptimizeForNonNullColumn);
     }
+
+    @Override
+    public boolean passesKeyRangeFilter(Scan scan) {
+        if (scan.getAttribute(SCAN_START_ROW_SUFFIX) == null) {
+            // Scan from compaction.
+            return true;
+        }
+        byte[] startKey = currentRegion.getStartKey();
+        byte[] endKey = currentRegion.getEndKey();
+        // If the region start key is not the prefix of the scan start row then we can return
empty
+        // scanners. This is possible during merge where one of the child region scan should
not return any
+        // results as we go through merged region.
+        int prefixLength = scan.getStartRow().length - scan.getAttribute(SCAN_START_ROW_SUFFIX).length;
+        if (Bytes.compareTo(scan.getStartRow(), 0, prefixLength,
+            (startKey.length == 0 ? new byte[endKey.length] : startKey), 0,
+            (startKey.length == 0 ? endKey.length : startKey.length)) != 0) {
+            return false;
+        }
+        return true;
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
index 5241885..a72aa92 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java
@@ -22,6 +22,8 @@ import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -30,18 +32,19 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.CompareOperator;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.MetaTableAccessor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoUtil;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.RegionObserver;
 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
@@ -68,11 +71,15 @@ import org.apache.phoenix.util.RepairUtil;
 
 import com.google.common.collect.Lists;
 
-public class IndexHalfStoreFileReaderGenerator implements RegionObserver {
+public class IndexHalfStoreFileReaderGenerator implements RegionObserver, RegionCoprocessor{
     
     private static final String LOCAL_INDEX_AUTOMATIC_REPAIR = "local.index.automatic.repair";
     public static final Log LOG = LogFactory.getLog(IndexHalfStoreFileReaderGenerator.class);
 
+    @Override
+    public Optional<RegionObserver> getRegionObserver() {
+      return Optional.of(this);
+    }
     
     @Override
     public StoreFileReader preStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment>
ctx,
@@ -82,7 +89,6 @@ public class IndexHalfStoreFileReaderGenerator implements RegionObserver
{
         Region region = ctx.getEnvironment().getRegion();
         RegionInfo childRegion = region.getRegionInfo();
         byte[] splitKey = null;
-        
         if (reader == null && r != null) {
             if(!p.toString().contains(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX)) {
                 return reader;
@@ -96,11 +102,11 @@ public class IndexHalfStoreFileReaderGenerator implements RegionObserver
{
                 SingleColumnValueFilter scvf = null;
                 if (Reference.isTopFileRegion(r.getFileRegion())) {
                     scvf = new SingleColumnValueFilter(HConstants.CATALOG_FAMILY,
-                        HConstants.SPLITB_QUALIFIER, CompareOperator.EQUAL, ((HRegionInfo)region.getRegionInfo()).toByteArray());
+                        HConstants.SPLITB_QUALIFIER, CompareOperator.EQUAL, RegionInfoUtil.toByteArray(region.getRegionInfo()));
                     scvf.setFilterIfMissing(true);
                 } else {
                     scvf = new SingleColumnValueFilter(HConstants.CATALOG_FAMILY,
-                        HConstants.SPLITA_QUALIFIER, CompareOperator.EQUAL, ((HRegionInfo)region.getRegionInfo()).toByteArray());
+                        HConstants.SPLITA_QUALIFIER, CompareOperator.EQUAL, RegionInfoUtil.toByteArray(region.getRegionInfo()));
                     scvf.setFilterIfMissing(true);
                 }
                 if(scvf != null) scan.setFilter(scvf);
@@ -168,7 +174,8 @@ public class IndexHalfStoreFileReaderGenerator implements RegionObserver
{
                 return new IndexHalfStoreFileReader(fs, p, cacheConf, in, size, r, ctx
                         .getEnvironment().getConfiguration(), indexMaintainers, viewConstants,
                         childRegion, regionStartKeyInHFile, splitKey,
-                        childRegion.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID);
+                        childRegion.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID,
+                        new AtomicInteger(0), region.getRegionInfo());
             } catch (ClassNotFoundException e) {
                 throw new IOException(e);
             } catch (SQLException e) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexStoreFileScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexStoreFileScanner.java
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexStoreFileScanner.java
index c8ec499..000f9c7 100644
--- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexStoreFileScanner.java
+++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/LocalIndexStoreFileScanner.java
@@ -22,11 +22,14 @@ import java.util.Map.Entry;
 import java.util.Optional;
 
 import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilder;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
+import org.apache.hadoop.hbase.CellComparatorImpl;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.util.PhoenixKeyValueUtil;
@@ -37,16 +40,18 @@ public class LocalIndexStoreFileScanner extends StoreFileScanner{
 
     private IndexHalfStoreFileReader reader;
     private boolean changeBottomKeys;
+    private CellComparatorImpl comparator;
     @SuppressWarnings("deprecation")
     public LocalIndexStoreFileScanner(IndexHalfStoreFileReader reader, boolean cacheBlocks,
boolean pread,
             boolean isCompaction, long readPt, long scannerOrder,
             boolean canOptimizeForNonNullColumn) {
-        super(reader, reader.getScanner(cacheBlocks, pread, isCompaction), true, reader
+        super(reader, reader.getScanner(cacheBlocks, pread, isCompaction), !isCompaction,
reader
                 .getHFileReader().hasMVCCInfo(), readPt, scannerOrder, canOptimizeForNonNullColumn);
         this.reader = reader;
         this.changeBottomKeys =
                 this.reader.getRegionInfo().getStartKey().length == 0
                         && this.reader.getSplitRow().length != this.reader.getOffset();
+        this.comparator = (CellComparatorImpl)getComparator();
     }
 
     @Override
@@ -124,11 +129,10 @@ public class LocalIndexStoreFileScanner extends StoreFileScanner{
             if (firstKey.isPresent()) {
                 return false;
             }
-            byte[] fk = PhoenixKeyValueUtil.maybeCopyCell(firstKey.get()).getKey();
-            if (getComparator().compare(kv, firstKey.get()) <= 0) {
+            if (this.comparator.compare(kv, firstKey.get(), true) <= 0) {
                 return super.seekToPreviousRow(key);
             }
-            KeyValue replacedKey = getKeyPresentInHFiles(kv.getRowArray());
+            Cell replacedKey = getKeyPresentInHFiles(kv);
             boolean seekToPreviousRow = super.seekToPreviousRow(replacedKey);
             while(super.peek()!=null && !isSatisfiedMidKeyCondition(super.peek()))
{
                 seekToPreviousRow = super.seekToPreviousRow(super.peek());
@@ -137,8 +141,8 @@ public class LocalIndexStoreFileScanner extends StoreFileScanner{
         } else {
             // The equals sign isn't strictly necessary just here to be consistent with
             // seekTo
-            KeyValue splitKeyValue = KeyValueUtil.createKeyValueFromKey(reader.getSplitkey());
-            if (getComparator().compare(kv, splitKeyValue) >= 0) {
+            KeyValue splitKeyValue = new KeyValue.KeyOnlyKeyValue(reader.getSplitkey());
+            if (this.comparator.compare(kv, splitKeyValue, true) >= 0) {
                 boolean seekToPreviousRow = super.seekToPreviousRow(kv);
                 while(super.peek()!=null && !isSatisfiedMidKeyCondition(super.peek()))
{
                     seekToPreviousRow = super.seekToPreviousRow(super.peek());
@@ -198,22 +202,21 @@ public class LocalIndexStoreFileScanner extends StoreFileScanner{
      * @param key
      *
      */
-    private KeyValue getKeyPresentInHFiles(byte[] key) {
-        KeyValue keyValue = new KeyValue(key);
+    private KeyValue getKeyPresentInHFiles(Cell keyValue) {
         int rowLength = keyValue.getRowLength();
         int rowOffset = keyValue.getRowOffset();
 
         short length = (short) (rowLength - reader.getSplitRow().length + reader.getOffset());
         byte[] replacedKey =
-                new byte[length + key.length - (rowOffset + rowLength) + ROW_LENGTH_SIZE];
+                new byte[length + keyValue.getRowArray().length - (rowOffset + rowLength)
+ ROW_LENGTH_SIZE];
         System.arraycopy(Bytes.toBytes(length), 0, replacedKey, 0, ROW_LENGTH_SIZE);
         System.arraycopy(reader.getRegionStartKeyInHFile(), 0, replacedKey, ROW_LENGTH_SIZE,
reader.getOffset());
         System.arraycopy(keyValue.getRowArray(), keyValue.getRowOffset() + reader.getSplitRow().length,
             replacedKey, reader.getOffset() + ROW_LENGTH_SIZE, rowLength
                     - reader.getSplitRow().length);
-        System.arraycopy(key, rowOffset + rowLength, replacedKey,
+        System.arraycopy(keyValue.getRowArray(), rowOffset + rowLength, replacedKey,
             reader.getOffset() + keyValue.getRowLength() - reader.getSplitRow().length
-                    + ROW_LENGTH_SIZE, key.length - (rowOffset + rowLength));
+                    + ROW_LENGTH_SIZE, keyValue.getRowArray().length - (rowOffset + rowLength));
         return new KeyValue.KeyOnlyKeyValue(replacedKey);
     }
     
@@ -225,31 +228,30 @@ public class LocalIndexStoreFileScanner extends StoreFileScanner{
      * @throws IOException
      */
     public boolean seekOrReseek(Cell cell, boolean isSeek) throws IOException{
-        KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(cell);
-        KeyValue keyToSeek = kv;
-        KeyValue splitKeyValue = KeyValueUtil.createKeyValueFromKey(reader.getSplitkey());
+        Cell keyToSeek = cell;
+        KeyValue splitKeyValue = new KeyValue.KeyOnlyKeyValue(reader.getSplitkey());
         if (reader.isTop()) {
-            if(getComparator().compare(kv, splitKeyValue) < 0){
+            if(this.comparator.compare(cell, splitKeyValue, true) < 0){
                 if(!isSeek && realSeekDone()) {
                     return true;
                 }
                 return seekOrReseekToProperKey(isSeek, keyToSeek);
             }
-            keyToSeek = getKeyPresentInHFiles(kv.getRowArray());
+            keyToSeek = getKeyPresentInHFiles(cell);
             return seekOrReseekToProperKey(isSeek, keyToSeek);
         } else {
-            if (getComparator().compare(kv, splitKeyValue) >= 0) {
+            if (this.comparator.compare(cell, splitKeyValue, true) >= 0) {
                 close();
                 return false;
             }
             if(!isSeek && reader.getRegionInfo().getStartKey().length == 0 &&
reader.getSplitRow().length > reader.getRegionStartKeyInHFile().length) {
-                keyToSeek = getKeyPresentInHFiles(kv.getRowArray());
+                keyToSeek = getKeyPresentInHFiles(cell);
             }
         }
         return seekOrReseekToProperKey(isSeek, keyToSeek);
     }
 
-    private boolean seekOrReseekToProperKey(boolean isSeek, KeyValue kv)
+    private boolean seekOrReseekToProperKey(boolean isSeek, Cell kv)
             throws IOException {
         boolean seekOrReseek = isSeek ? super.seek(kv) : super.reseek(kv);
         while (seekOrReseek && super.peek() != null
@@ -269,5 +271,4 @@ public class LocalIndexStoreFileScanner extends StoreFileScanner{
                 reader.getSplitRow().length, lenOfRemainingKey);
         return keyReplacedStartKey;
     }
-
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index ee34a8a..5c2e639 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -166,7 +166,7 @@ abstract public class BaseScannerRegionObserver implements RegionObserver
{
             throw new DoNotRetryIOException(cause.getMessage(), cause);
         }
         if(isLocalIndex) {
-            ScanUtil.setupLocalIndexScan(scan, lowerInclusiveRegionKey, upperExclusiveRegionKey);
+            ScanUtil.setupLocalIndexScan(scan);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 44e7f33..88f07f6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1054,7 +1054,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
             if (!tableExist) {
                 if (newDesc.build().getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES)
!= null && Boolean.TRUE.equals(
                         PBoolean.INSTANCE.toObject(newDesc.build().getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES))))
{
-                    newDesc.setValue(Bytes.toBytes(TableDescriptorBuilder.SPLIT_POLICY),
Bytes.toBytes(IndexRegionSplitPolicy.class.getName()));
+                    newDesc.setRegionSplitPolicyClassName(IndexRegionSplitPolicy.class.getName());
                 }
                 // Remove the splitPolicy attribute to prevent HBASE-12570
                 if (isMetaTable) {
@@ -1078,7 +1078,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                      * server and compatible. This works around HBASE-12570 which causes
the cluster to be
                      * brought down.
                      */
-                    newDesc.setValue(TableDescriptorBuilder.SPLIT_POLICY, MetaDataSplitPolicy.class.getName());
+                    newDesc.setRegionSplitPolicyClassName(MetaDataSplitPolicy.class.getName());
                     modifyTable(physicalTableName, newDesc.build(), true);
                 }
                 return null;
@@ -1087,11 +1087,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                     checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES,
this.getProps()).getName());
                 } else {
                     for(Pair<byte[],Map<String,Object>> family: families) {
-                        if ((newDesc.build().getValue(TableDescriptorBuilder.SPLIT_POLICY)==null
|| !newDesc.build().getValue(TableDescriptorBuilder.SPLIT_POLICY).equals(
-                                IndexRegionSplitPolicy.class.getName()))
-                                && Bytes.toString(family.getFirst()).startsWith(
-                                        QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX))
{
-                            newDesc.setValue(TableDescriptorBuilder.SPLIT_POLICY, IndexRegionSplitPolicy.class.getName());
+                        if ((Bytes.toString(family.getFirst())
+                                .startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX)))
{
+                            newDesc.setRegionSplitPolicyClassName(IndexRegionSplitPolicy.class.getName());
                             break;
                         }
                     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 94ac19f..761a9ab 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -657,10 +657,9 @@ public class ScanUtil {
      * @param lowerInclusiveRegionKey
      * @param upperExclusiveRegionKey
      */
-    public static void setupLocalIndexScan(Scan scan, byte[] lowerInclusiveRegionKey,
-            byte[] upperExclusiveRegionKey) {
-        byte[] prefix = lowerInclusiveRegionKey.length == 0 ? new byte[upperExclusiveRegionKey.length]:
lowerInclusiveRegionKey;
-        int prefixLength = lowerInclusiveRegionKey.length == 0? upperExclusiveRegionKey.length:
lowerInclusiveRegionKey.length;
+    public static void setupLocalIndexScan(Scan scan) {
+        byte[] prefix = scan.getStartRow().length == 0 ? new byte[scan.getStopRow().length]:
scan.getStartRow();
+        int prefixLength = scan.getStartRow().length == 0? scan.getStopRow().length: scan.getStartRow().length;
         if(scan.getAttribute(SCAN_START_ROW_SUFFIX)!=null) {
             scan.setStartRow(ScanRanges.prefixKey(scan.getAttribute(SCAN_START_ROW_SUFFIX),
0, prefix, prefixLength));
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 76757b0..64045ae 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -56,7 +56,6 @@ import org.apache.phoenix.jdbc.PhoenixStatement;
 import org.apache.phoenix.parse.JoinTableNode.JoinType;
 import org.apache.phoenix.parse.ParseNodeFactory;
 import org.apache.phoenix.parse.SelectStatement;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.ColumnRef;
 import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PColumnImpl;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/31c674a8/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
index f6dd85c..3f37ed9 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/TestWALRecoveryCaching.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.client.Table;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.coprocessor.RegionObserver;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -102,7 +104,12 @@ public class TestWALRecoveryCaching {
   // -----------------------------------------------------------------------------------------------
   private static CountDownLatch allowIndexTableToRecover;
 
-  public static class IndexTableBlockingReplayObserver implements RegionObserver {
+  public static class IndexTableBlockingReplayObserver implements RegionObserver, RegionCoprocessor
{
+
+    @Override
+    public Optional<RegionObserver> getRegionObserver() {
+      return Optional.of(this);
+    }
 
     @Override
         public void preWALRestore(


Mime
View raw message