phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject [3/4] PHOENIX-21: Support indexes on multi-tenant views. Still needs more testing
Date Tue, 18 Feb 2014 10:04:57 GMT
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index 26009ca..0d72c87 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -43,6 +43,7 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PDatum;
 import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ViewType;
 import org.apache.phoenix.schema.PTableType;
 
 import com.google.common.collect.Lists;
@@ -85,7 +86,7 @@ public class QueryOptimizer {
         }
         PTable dataTable = dataPlan.getTableRef().getTable();
         List<PTable>indexes = Lists.newArrayList(dataTable.getIndexes());
-        if (indexes.isEmpty() || dataPlan.getTableRef().hasDynamicCols() || select.getHint().hasHint(Hint.NO_INDEX))
{
+        if (indexes.isEmpty() || dataPlan.isDegenerate() || dataPlan.getTableRef().hasDynamicCols()
|| select.getHint().hasHint(Hint.NO_INDEX)) {
             return dataPlan;
         }
         
@@ -110,7 +111,14 @@ public class QueryOptimizer {
             return hintedPlan;
         }
         for (PTable index : indexes) {
-            addPlan(statement, translatedIndexSelect, index, targetColumns, parallelIteratorFactory,
plans);
+            QueryPlan plan = addPlan(statement, translatedIndexSelect, index, targetColumns,
parallelIteratorFactory, dataPlan);
+            if (plan != null) {
+                // Query can't possibly return anything so just return this plan.
+                if (plan.isDegenerate()) {
+                    return plan;
+                }
+                plans.add(plan);
+            }
         }
         
         return chooseBestPlan(select, plans);
@@ -151,8 +159,9 @@ public class QueryOptimizer {
                 int indexPos = getIndexPosition(indexes, indexName);
                 if (indexPos >= 0) {
                     // Hinted index is applicable, so return it. It'll be the plan at position
1, after the data plan
-                    if (addPlan(statement, select, indexes.get(indexPos), targetColumns,
parallelIteratorFactory, plans)) {
-                        return plans.get(1);
+                    QueryPlan plan = addPlan(statement, select, indexes.get(indexPos), targetColumns,
parallelIteratorFactory, dataPlan);
+                    if (plan != null) {
+                        return plan;
                     }
                     indexes.remove(indexPos);
                 }
@@ -171,8 +180,7 @@ public class QueryOptimizer {
         return -1;
     }
     
-    private static boolean addPlan(PhoenixStatement statement, SelectStatement select, PTable
index, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory,
List<QueryPlan> plans) throws SQLException {
-        QueryPlan dataPlan = plans.get(0);
+    private static QueryPlan addPlan(PhoenixStatement statement, SelectStatement select,
PTable index, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory,
QueryPlan dataPlan) throws SQLException {
         int nColumns = dataPlan.getProjector().getColumnCount();
         String alias = '"' + dataPlan.getTableRef().getTableAlias() + '"'; // double quote
in case it's case sensitive
         String schemaName = dataPlan.getTableRef().getTable().getSchemaName().getString();
@@ -183,13 +191,15 @@ public class QueryOptimizer {
         try {
             SelectStatement indexSelect = FACTORY.select(select, tables);
             ColumnResolver resolver = FromCompiler.getResolver(indexSelect, statement.getConnection());
-            QueryCompiler compiler = new QueryCompiler(statement, indexSelect, resolver,
targetColumns, parallelIteratorFactory);
-            QueryPlan plan = compiler.compile();
-            // Checking the index status and number of columns handles the wildcard cases
correctly
-            // We can't check the status earlier, because the index table may be out-of-date.
-            if (plan.getTableRef().getTable().getIndexState() == PIndexState.ACTIVE &&
plan.getProjector().getColumnCount() == nColumns) {
-                plans.add(plan);
-                return true;
+            // Check index state of now potentially updated index table to make sure it's
active
+            if (PIndexState.ACTIVE.equals(resolver.getTables().get(0).getTable().getIndexState()))
{
+                QueryCompiler compiler = new QueryCompiler(statement, indexSelect, resolver,
targetColumns, parallelIteratorFactory);
+                QueryPlan plan = compiler.compile();
+                // Checking number of columns handles the wildcard cases correctly, as in
that case the index
+                // must contain all columns from the data table to be able to be used.
+                if (plan.getTableRef().getTable().getIndexState() == PIndexState.ACTIVE &&
plan.getProjector().getColumnCount() == nColumns) {
+                    return plan;
+                }
             }
         } catch (ColumnNotFoundException e) {
             /* Means that a column is being used that's not in our index.
@@ -198,7 +208,7 @@ public class QueryOptimizer {
              * the index table to the data table.
              */
         }
-        return false;
+        return null;
     }
     
     /**
@@ -215,9 +225,9 @@ public class QueryOptimizer {
      * @return
      */
     private QueryPlan chooseBestPlan(SelectStatement select, List<QueryPlan> plans)
{
-        QueryPlan firstPlan = plans.get(0);
+        final QueryPlan dataPlan = plans.get(0);
         if (plans.size() == 1) {
-            return firstPlan;
+            return dataPlan;
         }
         
         /**
@@ -253,12 +263,31 @@ public class QueryOptimizer {
         if (bestCandidates.isEmpty()) {
             bestCandidates.addAll(stillCandidates);
         }
+        
+        int nViewConstants = 0;
+        PTable dataTable = dataPlan.getTableRef().getTable();
+        if (dataTable.getViewType() == ViewType.UPDATABLE) {
+            for (PColumn column : dataTable.getColumns()) {
+                if (column.getViewConstant() != null) {
+                    nViewConstants++;
+                }
+            }
+        }
+        final int boundRanges = nViewConstants;
         final int comparisonOfDataVersusIndexTable = select.getHint().hasHint(Hint.USE_DATA_OVER_INDEX_TABLE)
? -1 : 1;
         Collections.sort(bestCandidates, new Comparator<QueryPlan>() {
 
             @Override
             public int compare(QueryPlan plan1, QueryPlan plan2) {
+                PTable table1 = plan1.getTableRef().getTable();
+                PTable table2 = plan2.getTableRef().getTable();
                 int c = plan2.getContext().getScanRanges().getRanges().size() - plan1.getContext().getScanRanges().getRanges().size();
+                // Account for potential view constants which are always bound
+                if (plan1 == dataPlan) { // plan2 is index plan. Ignore the viewIndexId if
present
+                    c += boundRanges - (table2.getViewIndexId() == null ? 0 : 1);
+                } else { // plan1 is index plan. Ignore the viewIndexId if present
+                    c -= boundRanges - (table1.getViewIndexId() == null ? 0 : 1);
+                }
                 if (c != 0) return c;
                 if (plan1.getGroupBy()!=null && plan2.getGroupBy()!=null) {
                     if (plan1.getGroupBy().isOrderPreserving() != plan2.getGroupBy().isOrderPreserving())
{
@@ -266,8 +295,6 @@ public class QueryOptimizer {
                     }
                 }
                 // Use smaller table (table with fewest kv columns)
-                PTable table1 = plan1.getTableRef().getTable();
-                PTable table2 = plan2.getTableRef().getTable();
                 c = (table1.getColumns().size() - table1.getPKColumns().size()) - (table2.getColumns().size()
- table2.getPKColumns().size());
                 if (c != 0) return c;
                 

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
b/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
index 325554b..44d5f34 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
@@ -19,6 +19,10 @@ package org.apache.phoenix.parse;
 
 public class CreateSequenceStatement extends MutableStatement {
 
+    public static CreateSequenceStatement create(TableName sequenceName) {
+        return new CreateSequenceStatement(sequenceName, null, null, null, true, 0);
+    }
+    
 	private final TableName sequenceName;
 	private final ParseNode startWith;
 	private final ParseNode incrementBy;

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
index 4636caf..f9bbea1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
@@ -48,6 +48,13 @@ public class LiteralParseNode extends TerminalParseNode {
         this.value = this.type == null ? null : this.type.toObject(value, this.type);
     }
 
+    public LiteralParseNode(Object value, PDataType type) {
+        this.type = type;
+        // This will make the value null if the value passed through represents null for
the given type.
+        // For example, an empty string is treated as a null.
+        this.value = this.type == null ? null : this.type.toObject(value, this.type);
+    }
+
     public PDataType getType() {
         return type;
     }

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index a2e8eaa..192db11 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -34,6 +34,7 @@ import org.apache.phoenix.compile.MutationPlan;
 import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
 import org.apache.phoenix.execute.MutationState;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.SequenceKey;
 
@@ -67,7 +68,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
     public MetaDataMutationResult getTable(byte[] tenantId, byte[] schemaName, byte[] tableName,
long tableTimestamp, long clientTimetamp) throws SQLException;
     public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[]
tableName, PTableType tableType, Map<String,Object> tableProps, List<Pair<byte[],Map<String,Object>>>
families, byte[][] splits) throws SQLException;
     public MetaDataMutationResult dropTable(List<Mutation> tableMetadata, PTableType
tableType) throws SQLException;
-    public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTableType
tableType, List<Pair<byte[],Map<String,Object>>> families) throws SQLException;
+    public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, List<Pair<byte[],Map<String,Object>>>
families, PTable table) throws SQLException;
     public MetaDataMutationResult dropColumn(List<Mutation> tableMetadata, PTableType
tableType) throws SQLException;
     public MetaDataMutationResult updateIndexState(List<Mutation> tableMetadata, String
parentTableName) throws SQLException;
     public MutationState updateData(MutationPlan plan) throws SQLException;

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 8b73e49..02e3f2b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 import org.apache.hadoop.hbase.client.Append;
@@ -80,13 +81,17 @@ import org.apache.phoenix.schema.EmptySequenceCacheException;
 import org.apache.phoenix.schema.MetaDataSplitPolicy;
 import org.apache.phoenix.schema.NewerTableAlreadyExistsException;
 import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PDataType;
 import org.apache.phoenix.schema.PMetaData;
 import org.apache.phoenix.schema.PMetaDataImpl;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.ReadOnlyTableException;
+import org.apache.phoenix.schema.SaltingUtil;
 import org.apache.phoenix.schema.Sequence;
 import org.apache.phoenix.schema.SequenceKey;
+import org.apache.phoenix.schema.TableAlreadyExistsException;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.JDBCUtil;
@@ -275,6 +280,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                   currentKey = regionLocation.getRegionInfo().getEndKey();
                 } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW));
                 return locations;
+            } catch (org.apache.hadoop.hbase.TableNotFoundException e) {
+                String fullName = Bytes.toString(tableName);
+                throw new TableNotFoundException(SchemaUtil.getSchemaNameFromFullName(fullName),
SchemaUtil.getTableNameFromFullName(fullName));
             } catch (IOException e) {
                 if (retryCount++ < maxRetryCount) { // One retry, in case split occurs
while navigating
                     reload = true;
@@ -522,7 +530,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                     if (tableType == PTableType.VIEW) {
                         String fullTableName = Bytes.toString(tableName);
                         throw new ReadOnlyTableException(
-                                "The HBase column families for a read-only table must already
exist",
+                                "The HBase column families for a VIEW must already exist",
                                 SchemaUtil.getSchemaNameFromFullName(fullTableName),
                                 SchemaUtil.getTableNameFromFullName(fullTableName),
                                 Bytes.toString(family.getFirst()));
@@ -530,9 +538,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                     columnDescriptor = generateColumnFamilyDescriptor(family, tableType );
                 } else {
                     columnDescriptor = new HColumnDescriptor(oldDescriptor);
-                    if (tableType != PTableType.VIEW) {
-                        modifyColumnFamilyDescriptor(columnDescriptor, family);
+                    // Don't attempt to make any metadata changes for a VIEW
+                    if (tableType == PTableType.VIEW) {
+                        return;
                     }
+                    modifyColumnFamilyDescriptor(columnDescriptor, family);
                 }
                 
                 if (columnDescriptor.equals(oldDescriptor)) {
@@ -573,12 +583,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
     /**
      * 
      * @param tableName
-     * @param familyNames
      * @param splits
+     * @param modifyExistingMetaData TODO
+     * @param familyNames
      * @return true if table was created and false if it already exists
      * @throws SQLException
      */
-    private boolean ensureTableCreated(byte[] tableName, PTableType tableType , Map<String,Object>
props, List<Pair<byte[],Map<String,Object>>> families, byte[][] splits)
throws SQLException {
+    private HTableDescriptor ensureTableCreated(byte[] tableName, PTableType tableType ,
Map<String,Object> props, List<Pair<byte[],Map<String,Object>>> families,
byte[][] splits, boolean modifyExistingMetaData) throws SQLException {
         HBaseAdmin admin = null;
         SQLException sqlE = null;
         HTableDescriptor existingDesc = null;
@@ -618,7 +629,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                 } catch (TableExistsException e) {
                     // We can ignore this, as it just means that another client beat us
                     // to creating the HBase metadata.
-                    return false;
+                    return null;
                 }
                 if (isMetaTable) {
                     checkClientServerCompatibility();
@@ -633,14 +644,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                     admin.modifyTable(tableName, newDesc);
                     admin.enableTable(tableName);
                 }
-                return true;
+                return null;
             } else {
-                if (existingDesc.equals(newDesc)) {
+                if (!modifyExistingMetaData || existingDesc.equals(newDesc)) {
                     // Table is already created. Note that the presplits are ignored in this
case
                     if (isMetaTable) {
                         checkClientServerCompatibility();
                     }
-                    return false;
+                    return existingDesc;
                 }
 
                 if (isMetaTable) {
@@ -656,7 +667,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                 // TODO: What if not all existing column families are present?
                 admin.modifyTable(tableName, newDesc);
                 admin.enableTable(tableName);
-                return false;
+                return newDesc;
             }
 
         } catch (IOException e) {
@@ -678,7 +689,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                 }
             }
         }
-        return true; // will never make it here
+        return null; // will never make it here
     }
 
     private static boolean isInvalidMutableIndexConfig(Long serverVersion) {
@@ -786,22 +797,95 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
         }
     }
 
+    private void ensureViewIndexTableCreated(byte[] physicalTableName, Map<String,Object>
tableProps, List<Pair<byte[],Map<String,Object>>> families, byte[][] splits,
long timestamp) throws SQLException {
+        Long maxFileSize = (Long)tableProps.get(HTableDescriptor.MAX_FILESIZE);
+        if (maxFileSize == null) {
+            maxFileSize = this.config.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE);
+        }
+        byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName);
+        
+        int indexMaxFileSizePerc;
+        // Get percentage to use from table props first and then fallback to config
+        Integer indexMaxFileSizePercProp = (Integer)tableProps.remove(QueryServices.INDEX_MAX_FILESIZE_PERC_ATTRIB);
+        if (indexMaxFileSizePercProp == null) {
+            indexMaxFileSizePerc = config.getInt(QueryServices.INDEX_MAX_FILESIZE_PERC_ATTRIB,
QueryServicesOptions.DEFAULT_INDEX_MAX_FILESIZE_PERC);
+        } else {
+            indexMaxFileSizePerc = indexMaxFileSizePercProp;
+        }
+        long indexMaxFileSize = maxFileSize * indexMaxFileSizePerc / 100;
+        tableProps.put(HTableDescriptor.MAX_FILESIZE, indexMaxFileSize);
+        tableProps.put(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_NAME, PDataType.TRUE_BYTES);
+        // Only use splits if table is salted, otherwise it may not be applicable
+        HTableDescriptor desc = ensureTableCreated(physicalIndexName, PTableType.TABLE, tableProps,
families, splits, false);
+        if (desc != null) {
+            if (!Boolean.TRUE.equals(PDataType.BOOLEAN.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES))))
{
+                String fullTableName = Bytes.toString(physicalIndexName);
+                throw new TableAlreadyExistsException(
+                        "Unable to create shared physical table for indexes on views.",
+                        SchemaUtil.getSchemaNameFromFullName(fullTableName),
+                        SchemaUtil.getTableNameFromFullName(fullTableName));
+            }
+        }
+    }
+
 
+    private boolean ensureViewIndexTableDropped(byte[] physicalTableName, long timestamp)
throws SQLException {
+        byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName);
+        HTableDescriptor desc = null;
+        HBaseAdmin admin = null;
+        boolean wasDeleted = false;
+        try {
+            admin = new HBaseAdmin(config);
+            try {
+                desc = admin.getTableDescriptor(physicalIndexName);
+                if (Boolean.TRUE.equals(PDataType.BOOLEAN.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES))))
{
+                    final ReadOnlyProps props = this.getProps();
+                    final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
+                    if (dropMetadata) {
+                        admin.disableTable(physicalIndexName);
+                        admin.deleteTable(physicalIndexName);
+                        wasDeleted = true;
+                    }
+                }
+            } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
+                // Ignore, as we may never have created a view index table
+            }
+        } catch (IOException e) {
+            throw ServerUtil.parseServerException(e); 
+        } finally {
+            try {
+                if (admin != null) admin.close();
+            } catch (IOException e) {
+                logger.warn("",e);
+            }
+        }
+        return wasDeleted;
+    }
+    
     @Override
     public MetaDataMutationResult createTable(final List<Mutation> tableMetaData, byte[]
physicalTableName, PTableType tableType,
             Map<String,Object> tableProps, final List<Pair<byte[],Map<String,Object>>>
families, byte[][] splits) throws SQLException {
         byte[][] rowKeyMetadata = new byte[3][];
-        Mutation m = tableMetaData.get(0);
+        Mutation m = MetaDataUtil.getPutOnlyTableHeaderRow(tableMetaData);
         byte[] key = m.getRow();
         SchemaUtil.getVarChars(key, rowKeyMetadata);
         byte[] tenantIdBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
         byte[] schemaBytes = rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
         byte[] tableBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
-        if (tableType != PTableType.VIEW || physicalTableName != null) {
-            if (physicalTableName == null) {
-                physicalTableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
+        byte[] tableName = physicalTableName != null ? physicalTableName : SchemaUtil.getTableNameAsBytes(schemaBytes,
tableBytes);
+        if ((tableType == PTableType.VIEW && physicalTableName != null) || (tableType
!= PTableType.VIEW && physicalTableName == null)) {
+            // For views this will ensure that metadata already exists
+            ensureTableCreated(tableName, tableType, tableProps, families, splits, true);
+        }
+
+        if (tableType == PTableType.INDEX && physicalTableName != null) { // Index
on view
+            // Physical index table created up front for multi tenant
+            // TODO: if viewIndexId is Short.MIN_VALUE, then we don't need to attempt to
create it
+            if (!MetaDataUtil.isMultiTenant(m)) {
+                ensureViewIndexTableCreated(physicalTableName, MetaDataUtil.getClientTimeStamp(m));
             }
-            ensureTableCreated(physicalTableName, tableType, tableProps, families, splits);
+        } else if (tableType == PTableType.TABLE && MetaDataUtil.isMultiTenant(m))
{ // Create view index table up front for multi tenant tables
+            ensureViewIndexTableCreated(tableName, tableProps, families, MetaDataUtil.isSalted(m)
? splits : null, MetaDataUtil.getClientTimeStamp(m));
         }
         
         byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
@@ -848,11 +932,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
         final MutationCode code = result.getMutationCode();
         switch(code) {
         case TABLE_ALREADY_EXISTS:
-            final ReadOnlyProps props = this.getProps();
-            final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
+            ReadOnlyProps props = this.getProps();
+            boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
             if (dropMetadata) {
                 dropTables(result.getTableNamesToDelete());
             }
+            if (tableType == PTableType.TABLE) {
+                byte[] physicalTableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
+                long timestamp = MetaDataUtil.getClientTimeStamp(tableMetaData);
+                ensureViewIndexTableDropped(physicalTableName, timestamp);
+            }
             break;
         default:
             break;
@@ -895,20 +984,76 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
         }
     }
 
+    private static Map<String,Object> createPropertiesMap(Map<ImmutableBytesWritable,ImmutableBytesWritable>
htableProps) {
+        Map<String,Object> props = Maps.newHashMapWithExpectedSize(htableProps.size());
+        for (Map.Entry<ImmutableBytesWritable,ImmutableBytesWritable> entry : htableProps.entrySet())
{
+            ImmutableBytesWritable key = entry.getKey();
+            ImmutableBytesWritable value = entry.getValue();
+            props.put(Bytes.toString(key.get(), key.getOffset(), key.getLength()), Bytes.toString(value.get(),
value.getOffset(), value.getLength()));
+        }
+        return props;
+    }
+    
+    private void ensureViewIndexTableCreated(byte[] physicalIndexTableName, long timestamp)
throws SQLException {
+        PTable table;
+        String name = Bytes.toString(
+                physicalIndexTableName, 
+                MetaDataUtil.VIEW_INDEX_TABLE_PREFIX_BYTES.length,
+                physicalIndexTableName.length-MetaDataUtil.VIEW_INDEX_TABLE_PREFIX_BYTES.length);
+        try {
+            table = latestMetaData.getTable(name);
+            if (table.getTimeStamp() >= timestamp) { // Table in cache is newer than client
timestamp which shouldn't be the case
+                throw new TableNotFoundException(table.getSchemaName().getString(), table.getTableName().getString());
+            }
+        } catch (TableNotFoundException e) {
+            byte[] schemaName = Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(name));
+            byte[] tableName = Bytes.toBytes(SchemaUtil.getTableNameFromFullName(name));
+            MetaDataMutationResult result = this.getTable(null, schemaName, tableName, HConstants.LATEST_TIMESTAMP,
timestamp);
+            table = result.getTable();
+            if (table == null) {
+                throw e;
+            }
+        }
+        ensureViewIndexTableCreated(table, timestamp);
+    }
+    
+    private void ensureViewIndexTableCreated(PTable table, long timestamp) throws SQLException
{
+        byte[] physicalTableName = table.getPhysicalName().getBytes();
+        HTableDescriptor htableDesc = this.getTableDescriptor(physicalTableName);
+        Map<String,Object> tableProps = createPropertiesMap(htableDesc.getValues());
+        List<Pair<byte[],Map<String,Object>>> families = Lists.newArrayListWithExpectedSize(Math.max(1,
table.getColumnFamilies().size()));
+        if (families.isEmpty()) {
+            byte[] familyName = SchemaUtil.getEmptyColumnFamily(table);
+            Map<String,Object> familyProps = createPropertiesMap(htableDesc.getFamily(familyName).getValues());
+            families.add(new Pair<byte[],Map<String,Object>>(familyName, familyProps));
+        } else {
+            for (PColumnFamily family : table.getColumnFamilies()) {
+                byte[] familyName = SchemaUtil.getEmptyColumnFamily(table);
+                Map<String,Object> familyProps = createPropertiesMap(htableDesc.getFamily(familyName).getValues());
+                families.add(new Pair<byte[],Map<String,Object>>(family.getName().getBytes(),
familyProps));
+            }
+        }
+        byte[][] splits = null;
+        if (table.getBucketNum() != null) {
+            splits = SaltingUtil.getSalteByteSplitPoints(table.getBucketNum());
+        }
+        
+        ensureViewIndexTableCreated(physicalTableName, tableProps, families, splits, timestamp);
+    }
+    
     @Override
-    public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, PTableType
tableType, List<Pair<byte[],Map<String,Object>>> families) throws SQLException
{
+    public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, List<Pair<byte[],Map<String,Object>>>
families, PTable table) throws SQLException {
         byte[][] rowKeyMetaData = new byte[3][];
+        PTableType tableType = table.getType();
 
-        byte[] rowKey = tableMetaData.get(0).getRow();
+        Mutation m = tableMetaData.get(0);
+        byte[] rowKey = m.getRow();
         SchemaUtil.getVarChars(rowKey, rowKeyMetaData);
         byte[] tenantIdBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
         byte[] schemaBytes = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
         byte[] tableBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
         byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
-        byte[] tableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
         for ( Pair<byte[],Map<String,Object>>  family : families) {
-            
-            PTable table = latestMetaData.getTable(Bytes.toString(tableName));
             ensureFamilyCreated(table.getPhysicalName().getBytes(), tableType, family);
         }
         // Special case for call during drop table to ensure that the empty column family
exists.
@@ -926,6 +1071,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices
implement
                     return instance.addColumn(tableMetaData);
                 }
             });
+        if (tableType == PTableType.TABLE && result.getMutationCode() == MutationCode.COLUMN_NOT_FOUND)
{ // Success
+            // If we're changing MULTI_TENANT to true or false, create or drop the view index
table
+            KeyValue kv = MetaDataUtil.getMutationKeyValue(m, PhoenixDatabaseMetaData.MULTI_TENANT_BYTES);
+            if (kv != null) {
+                long timestamp = MetaDataUtil.getClientTimeStamp(m);
+                if (Boolean.TRUE.equals(PDataType.BOOLEAN.toObject(kv.getBuffer(), kv.getValueOffset(),
kv.getValueLength()))) {
+                    this.ensureViewIndexTableCreated(table, timestamp);
+                } else {
+                    this.ensureViewIndexTableDropped(table.getPhysicalName().getBytes(),
timestamp);
+                }
+            }
+        }
         return result;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 59761f1..6927d0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -176,7 +176,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices
imple
     }
 
     @Override
-    public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTableType
readOnly, List<Pair<byte[],Map<String,Object>>> families) throws SQLException
{
+    public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, List<Pair<byte[],Map<String,Object>>>
families, PTable table) throws SQLException {
         return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, null);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 7d92ab3..f586c9e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -118,8 +118,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices
imple
     }
 
     @Override
-    public MetaDataMutationResult addColumn(List<Mutation> tabeMetaData, PTableType
tableType, List<Pair<byte[],Map<String,Object>>> families ) throws SQLException
{
-        return getDelegate().addColumn(tabeMetaData, tableType, families);
+    public MetaDataMutationResult addColumn(List<Mutation> tabeMetaData, List<Pair<byte[],Map<String,Object>>>
families, PTable table) throws SQLException {
+        return getDelegate().addColumn(tabeMetaData, families, table);
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index b62c3e9..4987158 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -24,7 +24,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CURRENT_VALUE;
@@ -53,6 +52,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_TABLE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SELF_REFERENCING_COL_NAME_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB;
@@ -67,6 +67,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SCHEMA;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
 
@@ -167,6 +169,16 @@ public interface QueryConstants {
             REF_GENERATION_NAME + " VARCHAR," +
             TABLE_SEQ_NUM + " BIGINT," +
             COLUMN_COUNT + " INTEGER," +
+            SALT_BUCKETS + " INTEGER," +
+            DATA_TABLE_NAME + " VARCHAR," +
+            INDEX_STATE + " CHAR(1),\n" +
+            IMMUTABLE_ROWS + " BOOLEAN,\n" +
+            VIEW_STATEMENT + " VARCHAR,\n" +
+            DEFAULT_COLUMN_FAMILY_NAME + " VARCHAR,\n" +
+            DISABLE_WAL + " BOOLEAN,\n" +
+            MULTI_TENANT + " BOOLEAN,\n" +
+            VIEW_TYPE + " UNSIGNED_TINYINT,\n" +
+            VIEW_INDEX_ID + " SMALLINT,\n" +
             // Column metadata (will be null for table row)
             COLUMN_SIZE + " INTEGER," +
             BUFFER_LENGTH + " INTEGER," +
@@ -182,23 +194,13 @@ public interface QueryConstants {
             SCOPE_CATALOG + " VARCHAR," +
             SCOPE_SCHEMA + " VARCHAR," +
             SCOPE_TABLE + " VARCHAR," +
-            SOURCE_DATA_TYPE + " INTEGER," + // supposed to be SHORT
+            SOURCE_DATA_TYPE + " SMALLINT," +
             IS_AUTOINCREMENT + " VARCHAR," +
-            // Columns added in 1.2.1
             SORT_ORDER + " INTEGER," +
-            SALT_BUCKETS + " INTEGER," +
-            // Columns added in 2.0.0
-            DATA_TABLE_NAME + " VARCHAR," +
-            INDEX_STATE + " CHAR(1),\n" +
-            IMMUTABLE_ROWS + " BOOLEAN,\n" +
-            // Columns added in 3.0.0
-            VIEW_STATEMENT + " VARCHAR,\n" +
-            DEFAULT_COLUMN_FAMILY_NAME + " VARCHAR,\n" +
-            DISABLE_WAL + " BOOLEAN,\n" +
-            MULTI_TENANT + " BOOLEAN,\n" +
-            VIEW_TYPE + " UNSIGNED_TINYINT,\n" +
-            LINK_TYPE + " UNSIGNED_TINYINT,\n" +
             ARRAY_SIZE + " INTEGER,\n" +
+            VIEW_CONSTANT + " VARBINARY,\n" +
+            // Link metadata (only set on rows linking table to index or view)
+            LINK_TYPE + " UNSIGNED_TINYINT,\n" +
             "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
             + TABLE_SCHEM_NAME + "," + TABLE_NAME_NAME + "," + COLUMN_NAME + "," + TABLE_CAT_NAME
+ "))\n" +
             HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS +
",\n" +

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index b91068a..b6a65c0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.query;
 import java.util.concurrent.ExecutorService;
 
 import org.apache.http.annotation.Immutable;
-
 import org.apache.phoenix.iterate.SpoolTooBigToDiskException;
 import org.apache.phoenix.memory.MemoryManager;
 import org.apache.phoenix.optimize.QueryOptimizer;
@@ -93,6 +92,7 @@ public interface QueryServices extends SQLCloseable {
     public static final String ZOOKEEPER_ROOT_NODE_ATTRIB = "zookeeper.znode.parent";
     public static final String DISTINCT_VALUE_COMPRESS_THRESHOLD_ATTRIB = "phoenix.distinct.value.compress.threshold";
     public static final String SEQUENCE_CACHE_SIZE_ATTRIB = "phoenix.sequence.cacheSize";
+    public static final String INDEX_MAX_FILESIZE_PERC_ATTRIB = "phoenix.index.maxDataFileSizePerc";
 
     
     /**

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 4a07245..f9fdbfe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -22,8 +22,8 @@ import static org.apache.phoenix.query.QueryServices.CALL_QUEUE_ROUND_ROBIN_ATTR
 import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.GROUPBY_MAX_CACHE_SIZE_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILL_FILES_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILLABLE_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILL_FILES_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.IMMUTABLE_ROWS_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.KEEP_ALIVE_MS_ATTRIB;
@@ -57,7 +57,6 @@ import java.util.Map.Entry;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec;
-
 import org.apache.phoenix.util.DateUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
 
@@ -111,6 +110,7 @@ public class QueryServicesOptions {
     public static final long DEFAULT_GROUPBY_MAX_CACHE_MAX = 1024L*1024L*100L;  // 100 Mb
     
     public static final int DEFAULT_SEQUENCE_CACHE_SIZE = 100;  // reserve 100 sequences
at a time
+    public static final int DEFAULT_INDEX_MAX_FILESIZE_PERC = 50; // % of data table max
file size for index table
     
     
     private final Configuration config;

http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
index c916555..aa2fb3e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
@@ -66,4 +66,9 @@ public class DelegateColumn extends DelegateDatum implements PColumn {
     public Integer getArraySize() {
         return getDelegate().getArraySize();
     }
+
+    @Override
+    public byte[] getViewConstant() {
+        return getDelegate().getViewConstant();
+    }
 }


Mime
View raw message