phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sama...@apache.org
Subject phoenix git commit: Fix column qualifiers for StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL. Enable encoded column qualifiers for joins. Bring back qualifier counter per column family. Please enter the commit message for your changes. Lines starting
Date Fri, 23 Sep 2016 17:29:59 GMT
Repository: phoenix
Updated Branches:
  refs/heads/encodecolumns 29dbf051c -> aaecd5a4d


Fix column qualifiers for StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL.
Enable encoded column qualifiers for joins.
Bring back qualifier counter per column family.
 Please enter the commit message for your changes. Lines starting


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/aaecd5a4
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/aaecd5a4
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/aaecd5a4

Branch: refs/heads/encodecolumns
Commit: aaecd5a4d9c12bcaad2d278b79a87e4ddc888671
Parents: 29dbf05
Author: Samarth <samarth.jain@salesforce.com>
Authored: Fri Sep 23 10:29:02 2016 -0700
Committer: Samarth <samarth.jain@salesforce.com>
Committed: Fri Sep 23 10:29:02 2016 -0700

----------------------------------------------------------------------
 .../phoenix/compile/TupleProjectionCompiler.java   | 17 ++++-------------
 .../phoenix/expression/ArrayColumnExpression.java  |  8 +++++---
 .../org/apache/phoenix/schema/MetaDataClient.java  |  9 ++++-----
 .../java/org/apache/phoenix/schema/PTable.java     | 15 +++++++++++++++
 .../java/org/apache/phoenix/schema/PTableImpl.java |  2 +-
 .../tuple/EncodedColumnQualiferCellsList.java      |  4 ++--
 .../java/org/apache/phoenix/util/ScanUtil.java     |  2 +-
 .../apache/phoenix/execute/CorrelatePlanTest.java  |  4 +++-
 .../execute/LiteralResultIteratorPlanTest.java     |  4 +++-
 9 files changed, 38 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 3898969..9f7de36 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -26,7 +26,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.parse.AliasedNode;
 import org.apache.phoenix.parse.ColumnParseNode;
 import org.apache.phoenix.parse.FamilyWildcardParseNode;
@@ -168,8 +167,6 @@ public class TupleProjectionCompiler {
         boolean hasSaltingColumn = retainPKColumns && table.getBucketNum() != null;
         List<PColumn> projectedColumns = new ArrayList<PColumn>();
         int position = hasSaltingColumn ? 1 : 0;
-        EncodedCQCounter cqCounter = null;
-        String valueColFamilyName = Bytes.toString(VALUE_COLUMN_FAMILY);
         for (int i = position; i < sourceColumnRefs.size(); i++) {
             ColumnRef sourceColumnRef = sourceColumnRefs.get(i);
             PColumn sourceColumn = sourceColumnRef.getColumn();
@@ -182,18 +179,12 @@ public class TupleProjectionCompiler {
                     retainPKColumns && SchemaUtil.isPKColumn(sourceColumn) ? 
                             null : PNameFactory.newName(VALUE_COLUMN_FAMILY), 
                     position++, sourceColumn.isNullable(), sourceColumnRef);
-            if (EncodedColumnsUtil.hasEncodedColumnName(sourceColumn)) {
-                if (cqCounter == null) {
-                    cqCounter = new EncodedCQCounter();
-                } else {
-                    cqCounter.increment(valueColFamilyName);
-                }
-            }
             projectedColumns.add(column);
         }
-        StorageScheme storageScheme = StorageScheme.NON_ENCODED_COLUMN_NAMES;
-        if (cqCounter != null) {
-            storageScheme = StorageScheme.ENCODED_COLUMN_NAMES;
+        StorageScheme storageScheme = table.getStorageScheme();
+        EncodedCQCounter cqCounter = EncodedCQCounter.NULL_COUNTER;
+        if (storageScheme != StorageScheme.NON_ENCODED_COLUMN_NAMES) {
+            cqCounter = EncodedCQCounter.copy(table.getEncodedCQCounter());
         }
         return PTableImpl.makePTable(table.getTenantId(), PROJECTED_TABLE_SCHEMA, table.getName(),
PTableType.PROJECTED,
                 null, table.getTimeStamp(), table.getSequenceNumber(), table.getPKName(),

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
index aaa9f75..7310e59 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java
@@ -30,6 +30,7 @@ import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PArrayDataType;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 
 /**
  * 
@@ -57,10 +58,11 @@ public class ArrayColumnExpression extends ColumnExpression {
     
     public ArrayColumnExpression(PColumn column, String displayName, boolean encodedColumnName)
{
         super(column);
-        // array indexes are 1-based
-        this.index = column.getEncodedColumnQualifier()+1;
+        // array indexes are 1-based TODO: samarth think about the case when the encodedcolumn
qualifier is null. Probably best to add a check here for encodedcolumnname to be true
+        this.index = column.getEncodedColumnQualifier() + 1;
         byte[] cf = column.getFamilyName().getBytes();
-        this.arrayExpression = new KeyValueColumnExpression(column, cf, cf);
+        byte[] cq = EncodedColumnsUtil.getEncodedColumnQualifier(column);
+        this.arrayExpression = new KeyValueColumnExpression(column, cf, cq);
         this.origKVExpression = new KeyValueColumnExpression(column, displayName, encodedColumnName);
         this.displayName = displayName;
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 5f160cc..beaa376 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -2052,8 +2052,8 @@ public class MetaDataClient {
                  * in the client cache. If the phoenix table already doesn't exist then the
non-encoded column qualifier scheme works
                  * because we cannot control the column qualifiers that were used when populating
the hbase table.
                  */
-                //TODO: samarth these checks need to be changed for local indexes. Hate having
all of these special cases for local indexes.
-                // It is making the code unmaintainable.
+                //TODO: samarth these checks for whether table exists need to be changed
for local indexes. Hate having all of these special cases for local indexes.
+                // It is making the code unmaintainable. For local indexes, the physical
table already exists.
                 byte[] tableNameBytes = SchemaUtil.getTableNameAsBytes(schemaName, tableName);
                 boolean tableExists = true;
                 try (HBaseAdmin admin = connection.getQueryServices().getAdmin()) {
@@ -2072,7 +2072,6 @@ public class MetaDataClient {
                 } else if (tableExists) {
                     storageScheme = StorageScheme.NON_ENCODED_COLUMN_NAMES;
                 } else if (isImmutableRows) {
-//                    storageScheme = StorageScheme.NON_ENCODED_COLUMN_NAMES;
                     storageScheme = StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL;
                     // since we are storing all columns of a column family in a single key
value we can't use deletes to store nulls
                     storeNulls = true;
@@ -2147,14 +2146,14 @@ public class MetaDataClient {
                 }
             }
             
-            if (storageScheme == StorageScheme.ENCODED_COLUMN_NAMES) {
+            if (EncodedColumnsUtil.usesEncodedColumnNames(storageScheme)) {
                 // Store the encoded column counter for phoenix entities that have their
own hbase
                 // tables i.e. base tables and indexes.
                 String schemaNameToUse = tableType == VIEW ? viewPhysicalTable.getSchemaName().getString()
: schemaName;
                 String tableNameToUse = tableType == VIEW ? viewPhysicalTable.getTableName().getString()
: tableName;
+                boolean sharedIndex = tableType == PTableType.INDEX && (indexType
== IndexType.LOCAL || parent.getType() == PTableType.VIEW);
                 // For local indexes and indexes on views, pass on the the tenant id since
all their meta-data rows have
                 // tenant ids in there.
-                boolean sharedIndex = tableType == PTableType.INDEX && (indexType
== IndexType.LOCAL || parent.getType() == PTableType.VIEW);
                 String tenantIdToUse = connection.getTenantId() != null && sharedIndex
? connection.getTenantId().getString() : null;
                 // When a view adds its own columns, then we need to increase the sequence
number of the base table
                 // too since we want clients to get the latest PTable of the base table.

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index ee8a268..13ad1c7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import javax.annotation.Nullable;
 
@@ -31,6 +32,7 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryConstants;
 
 
 /**
@@ -411,6 +413,19 @@ public interface PTable extends PMetaDataEntity {
         
         private final Map<String, Integer> familyCounters = new HashMap<>();
         
+        /**
+         * Copy constructor
+         * @param counterToCopy
+         * @return copy of the passed counter
+         */
+        public static EncodedCQCounter copy(EncodedCQCounter counterToCopy) {
+            EncodedCQCounter cqCounter = new EncodedCQCounter();
+            for (Entry<String, Integer> e : counterToCopy.values().entrySet()) {
+                cqCounter.setValue(e.getKey(), e.getValue());
+            }
+            return cqCounter;
+        }
+        
         public static final EncodedCQCounter NULL_COUNTER = new EncodedCQCounter() {
             
             @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 63c1c59..ed26f17 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -1226,7 +1226,7 @@ public class PTableImpl implements PTable {
         if (table.hasStorageScheme()) {
             storageScheme = StorageScheme.fromSerializedValue(table.getStorageScheme().toByteArray()[0]);
         }
-        EncodedCQCounter encodedColumnQualifierCounter = storageScheme == StorageScheme.ENCODED_COLUMN_NAMES
? new EncodedCQCounter() : EncodedCQCounter.NULL_COUNTER;
+        EncodedCQCounter encodedColumnQualifierCounter = EncodedColumnsUtil.usesEncodedColumnNames(storageScheme)
? new EncodedCQCounter() : EncodedCQCounter.NULL_COUNTER;
         if (table.getEncodedCQCountersList() != null) {
             encodedColumnQualifierCounter = new EncodedCQCounter();
             for (org.apache.phoenix.coprocessor.generated.PTableProtos.EncodedCQCounter cqCounterFromProto
: table.getEncodedCQCountersList()) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
index 2110125..4caabbb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
@@ -32,9 +32,7 @@ import javax.annotation.concurrent.NotThreadSafe;
 
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.PTable.StorageScheme;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.types.PInteger;
@@ -49,6 +47,8 @@ import org.apache.phoenix.schema.types.PInteger;
  * implements all the optional methods of the interface too.
  * 
  * For getting elements out o
+ * 
+ * TODO: samarth think about whether we should be using this list with other storage schemes
  */
 @NotThreadSafe
 public class EncodedColumnQualiferCellsList implements List<Cell> {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 00ad1ae..80cb074 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -915,7 +915,7 @@ public class ScanUtil {
     }
     
     public static boolean useQualifierAsIndex(Pair<Integer, Integer> minMaxQualifiers,
boolean isJoin) {
-        return minMaxQualifiers != null && !isJoin;
+        return minMaxQualifiers != null;// && isJoin;
     }
     
     public static boolean setMinMaxQualifiersOnScan(PTable table) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 680dc57..ca6a471 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -63,6 +63,8 @@ import org.apache.phoenix.schema.PColumnImpl;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
+import org.apache.phoenix.schema.PTable.StorageScheme;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableRef;
@@ -258,7 +260,7 @@ public class CorrelatePlanTest {
                     PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
                     null, null, columns, null, null, Collections.<PTable>emptyList(),
                     false, Collections.<PName>emptyList(), null, null, false, false,
false, null,
-                    null, null, true, false, 0, 0L, Boolean.FALSE, null, false, null, null);
+                    null, null, true, false, 0, 0L, Boolean.FALSE, null, false, StorageScheme.NON_ENCODED_COLUMN_NAMES,
EncodedCQCounter.NULL_COUNTER);
             TableRef sourceTable = new TableRef(pTable);
             List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
             for (PColumn column : sourceTable.getTable().getColumns()) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/aaecd5a4/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
index bae4dd5..7fff2c4 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
@@ -57,9 +57,11 @@ import org.apache.phoenix.schema.PColumnImpl;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.PTable.StorageScheme;
 import org.apache.phoenix.schema.tuple.SingleKeyValueTuple;
 import org.apache.phoenix.schema.tuple.Tuple;
 import org.junit.Test;
@@ -179,7 +181,7 @@ public class LiteralResultIteratorPlanTest {
             PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME,
PTableType.SUBQUERY, null,
                     MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null,
columns, null, null,
                     Collections.<PTable> emptyList(), false, Collections.<PName>
emptyList(), null, null, false, false,
-                    false, null, null, null, true, false, 0, 0L, false, null, false, null,
null);
+                    false, null, null, null, true, false, 0, 0L, false, null, false, StorageScheme.NON_ENCODED_COLUMN_NAMES,
EncodedCQCounter.NULL_COUNTER);
             TableRef sourceTable = new TableRef(pTable);
             List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
             for (PColumn column : sourceTable.getTable().getColumns()) {


Mime
View raw message