phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject [04/20] phoenix git commit: PHOENIX-4290 Full table scan performed for DELETE with table having immutable indexes
Date Thu, 09 Nov 2017 20:58:12 GMT
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index 0d06f0a..c84e1d7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -17,8 +17,6 @@
  */
 package org.apache.phoenix.compile;
 
-import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
-
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
 import java.util.ArrayList;
@@ -66,6 +64,7 @@ import org.apache.phoenix.schema.ColumnNotFoundException;
 import org.apache.phoenix.schema.ColumnRef;
 import org.apache.phoenix.schema.FunctionNotFoundException;
 import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.MetaDataEntityNotFoundException;
 import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PColumnFamily;
 import org.apache.phoenix.schema.PColumnFamilyImpl;
@@ -73,9 +72,9 @@ import org.apache.phoenix.schema.PColumnImpl;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
-import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.PTableType;
@@ -871,7 +870,9 @@ public class FromCompiler {
                     TableRef tableRef = iterator.next();
                     try {
                         PColumnFamily columnFamily = tableRef.getTable().getColumnFamily(cfName);
-                        if (theColumnFamilyRef != null) { throw new TableNotFoundException(cfName);
}
+                        if (columnFamily == null) { 
+                            throw new TableNotFoundException(cfName); 
+                        }
                         theColumnFamilyRef = new ColumnFamilyRef(tableRef, columnFamily);
                     } catch (ColumnFamilyNotFoundException e) {}
                 }
@@ -914,10 +915,42 @@ public class FromCompiler {
                     PColumn column = tableRef.getTable().getColumnForColumnName(colName);
                     return new ColumnRef(tableRef, column.getPosition());
                 } catch (TableNotFoundException e) {
-                    // Try using the tableName as a columnFamily reference instead
-                    ColumnFamilyRef cfRef = resolveColumnFamily(schemaName, tableName);
-                    PColumn column = cfRef.getFamily().getPColumnForColumnName(colName);
-                    return new ColumnRef(cfRef.getTableRef(), column.getPosition());
+                    TableRef theTableRef = null;
+                    PColumn theColumn = null;
+                    PColumnFamily theColumnFamily = null;
+                    if (schemaName != null) {
+                        try {
+                            // Try schemaName as the tableName and use tableName as column
family name
+                            theTableRef = resolveTable(null, schemaName);
+                            theColumnFamily = theTableRef.getTable().getColumnFamily(tableName);
+                            theColumn = theColumnFamily.getPColumnForColumnName(colName);
+                        } catch (MetaDataEntityNotFoundException e2) {
+                        }
+                    } 
+                    if (theColumn == null) {
+                        // Try using the tableName as a columnFamily reference instead
+                        // and resolve column in each column family.
+                        Iterator<TableRef> iterator = tables.iterator();
+                        while (iterator.hasNext()) {
+                            TableRef tableRef = iterator.next();
+                            try {
+                                PColumnFamily columnFamily = tableRef.getTable().getColumnFamily(tableName);
+                                PColumn column = columnFamily.getPColumnForColumnName(colName);
+                                if (theColumn != null) {
+                                    throw new AmbiguousColumnException(colName);
+                                }
+                                theTableRef = tableRef;
+                                theColumnFamily = columnFamily;
+                                theColumn = column;
+                            } catch (MetaDataEntityNotFoundException e1) {
+                            }
+                        }
+                        if (theColumn == null) { 
+                            throw new ColumnNotFoundException(colName);
+                        }
+                    }
+                    ColumnFamilyRef cfRef = new ColumnFamilyRef(theTableRef, theColumnFamily);
+                    return new ColumnRef(cfRef.getTableRef(), theColumn.getPosition());
                 }
             }
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 4ebca90..796dad0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -153,7 +153,7 @@ public class TupleProjectionCompiler {
                 PTableType.PROJECTED, table.getIndexState(), table.getTimeStamp(), table.getSequenceNumber(),
                 table.getPKName(), table.getBucketNum(), projectedColumns, table.getParentSchemaName(),
                 table.getParentTableName(), table.getIndexes(), table.isImmutableRows(),
Collections.<PName> emptyList(),
-                null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(),
table.getViewType(),
+                table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(),
table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
                 table.getViewIndexId(),
                 table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional(),
table.getUpdateCacheFrequency(), 
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(),
table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(),
table.getEncodedCQCounter(), table.useStatsForParallelization());

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index f25f7f1..cfeb212 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -244,7 +244,6 @@ public enum SQLExceptionCode {
     SET_UNSUPPORTED_PROP_ON_ALTER_TABLE(1025, "42Y83", "Unsupported property set in ALTER
TABLE command."),
     CANNOT_ADD_NOT_NULLABLE_COLUMN(1038, "42Y84", "Only nullable columns may be added for
a pre-existing table."),
     NO_MUTABLE_INDEXES(1026, "42Y85", "Mutable secondary indexes are only supported for HBase
version " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD)
+ " and above."),
-    INVALID_FILTER_ON_IMMUTABLE_ROWS(1027, "42Y86", "All columns referenced in a WHERE clause
must be available in every index for a table with immutable rows."),
     INVALID_INDEX_STATE_TRANSITION(1028, "42Y87", "Invalid index state transition."),
     INVALID_MUTABLE_INDEX_CONFIG(1029, "42Y88", "Mutable secondary indexes must have the
"
             + IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY + " property set to "

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index bd0743c..b0974c6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -190,7 +190,9 @@ public class MutationState implements SQLCloseable {
 
     public MutationState(TableRef table, Map<ImmutableBytesPtr,RowMutationState> mutations,
long sizeOffset, long maxSize, long maxSizeBytes, PhoenixConnection connection) throws SQLException
{
         this(maxSize, maxSizeBytes, connection, false, null, sizeOffset);
-        this.mutations.put(table, mutations);
+        if (!mutations.isEmpty()) {
+            this.mutations.put(table, mutations);
+        }
         this.numRows = mutations.size();
         throwIfTooBig();
     }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index 2c55bdf..dc26d5a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -366,7 +366,6 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference>
{
     
     private IndexMaintainer(final PTable dataTable, final PTable index, PhoenixConnection
connection) {
         this(dataTable.getRowKeySchema(), dataTable.getBucketNum() != null);
-        assert(dataTable.getType() == PTableType.SYSTEM || dataTable.getType() == PTableType.TABLE
|| dataTable.getType() == PTableType.VIEW);
         this.rowKeyOrderOptimizable = index.rowKeyOrderOptimizable();
         this.isMultiTenant = dataTable.isMultiTenant();
         this.viewIndexId = index.getViewIndexId() == null ? null : MetaDataUtil.getViewIndexIdDataType().toBytes(index.getViewIndexId());
@@ -411,15 +410,14 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference>
{
         int nDataPKColumns = dataRowKeySchema.getFieldCount() - dataPosOffset;
         // For indexes on views, we need to remember which data columns are "constants"
         // These are the values in a VIEW where clause. For these, we don't put them in the
-        // index, as they're the same for every row in the index.
-        if (dataTable.getType() == PTableType.VIEW) {
-            List<PColumn>dataPKColumns = dataTable.getPKColumns();
-            for (int i = dataPosOffset; i < dataPKColumns.size(); i++) {
-                PColumn dataPKColumn = dataPKColumns.get(i);
-                if (dataPKColumn.getViewConstant() != null) {
-                    bitSet.set(i);
-                    nDataPKColumns--;
-                }
+        // index, as they're the same for every row in the index. The data table can be
+        // either a VIEW or PROJECTED
+        List<PColumn>dataPKColumns = dataTable.getPKColumns();
+        for (int i = dataPosOffset; i < dataPKColumns.size(); i++) {
+            PColumn dataPKColumn = dataPKColumns.get(i);
+            if (dataPKColumn.getViewConstant() != null) {
+                bitSet.set(i);
+                nDataPKColumns--;
             }
         }
         this.indexTableName = indexTableName;
@@ -543,11 +541,14 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference>
{
         for (int i = 0; i < index.getColumnFamilies().size(); i++) {
             PColumnFamily family = index.getColumnFamilies().get(i);
             for (PColumn indexColumn : family.getColumns()) {
-                PColumn dataColumn = IndexUtil.getDataColumn(dataTable, indexColumn.getName().getString());
-                byte[] dataColumnCq = dataColumn.getColumnQualifierBytes();
-                byte[] indexColumnCq = indexColumn.getColumnQualifierBytes();
-                this.coveredColumnsMap.put(new ColumnReference(dataColumn.getFamilyName().getBytes(),
dataColumnCq), 
-                        new ColumnReference(indexColumn.getFamilyName().getBytes(), indexColumnCq));
+                PColumn dataColumn = IndexUtil.getDataColumnOrNull(dataTable, indexColumn.getName().getString());
+                // This can happen during deletion where we don't need covered columns
+                if (dataColumn != null) {
+                    byte[] dataColumnCq = dataColumn.getColumnQualifierBytes();
+                    byte[] indexColumnCq = indexColumn.getColumnQualifierBytes();
+                    this.coveredColumnsMap.put(new ColumnReference(dataColumn.getFamilyName().getBytes(),
dataColumnCq), 
+                            new ColumnReference(indexColumn.getFamilyName().getBytes(), indexColumnCq));
+                }
             }
         }
         this.estimatedIndexRowKeyBytes = estimateIndexRowKeyByteSize(indexColByteSize);
@@ -758,8 +759,10 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference>
{
             int minLength = length - maxTrailingNulls;
             byte[] dataRowKey = stream.getBuffer();
             // Remove trailing nulls
-            while (length > minLength && dataRowKey[length-1] == QueryConstants.SEPARATOR_BYTE)
{
+            int index = dataRowKeySchema.getFieldCount() - 1;
+            while (index >= 0 && !dataRowKeySchema.getField(index).getDataType().isFixedWidth()
&& length > minLength && dataRowKey[length-1] == QueryConstants.SEPARATOR_BYTE)
{
                 length--;
+                index--;
             }
             // TODO: need to capture nDataSaltBuckets instead of just a boolean. For now,
             // we store this in nIndexSaltBuckets, as we only use this function for local
indexes

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index ca7ff2c..b3df50b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -429,7 +429,7 @@ public class QueryOptimizer {
             
         });
         
-        return bestCandidates;
+        return stopAtBestPlan ? bestCandidates.subList(0, 1) : bestCandidates;
     }
 
     

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 994b769..23b5161 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -295,6 +295,16 @@ public class PTableImpl implements PTable {
                 table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(),
table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(),
table.getEncodedCQCounter(), table.useStatsForParallelization());
     }
     
+    public static PTableImpl makePTable(PTable table, PTableType type, Collection<PColumn>
columns) throws SQLException {
+        return new PTableImpl(
+                table.getTenantId(), table.getSchemaName(), table.getTableName(), type, table.getIndexState(),
table.getTimeStamp(),
+                table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns,
table.getParentSchemaName(), table.getParentTableName(),
+                table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(),
table.getViewStatement(),
+                table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
table.getViewIndexId(), table.getIndexType(),
+                table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(),
table.getUpdateCacheFrequency(),
+                table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(),
table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(),
table.getEncodedCQCounter(), table.useStatsForParallelization());
+    }
+
     public static PTableImpl makePTable(PTable table, Collection<PColumn> columns,
PName defaultFamily) throws SQLException {
         return new PTableImpl(
                 table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(),
table.getIndexState(), table.getTimeStamp(),

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 36fa011..cacf4c4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -207,27 +207,35 @@ public class IndexUtil {
     }
     
     public static PColumn getDataColumn(PTable dataTable, String indexColumnName) {
+        PColumn column = getDataColumnOrNull(dataTable, indexColumnName);
+        if (column == null) {
+            throw new IllegalArgumentException("Could not find column \"" + SchemaUtil.getColumnName(getDataColumnFamilyName(indexColumnName),
getDataColumnName(indexColumnName)) + " in " + dataTable);
+        }
+        return column;
+    }
+    
+    public static PColumn getDataColumnOrNull(PTable dataTable, String indexColumnName) {
         int pos = indexColumnName.indexOf(INDEX_COLUMN_NAME_SEP);
         if (pos < 0) {
-            throw new IllegalArgumentException("Could not find expected '" + INDEX_COLUMN_NAME_SEP
+  "' separator in index column name of \"" + indexColumnName + "\"");
+            return null;
         }
         if (pos == 0) {
             try {
                 return dataTable.getPKColumn(indexColumnName.substring(1));
             } catch (ColumnNotFoundException e) {
-                throw new IllegalArgumentException("Could not find PK column \"" +  indexColumnName.substring(pos+1)
+ "\" in index column name of \"" + indexColumnName + "\"", e);
+                return null;
             }
         }
         PColumnFamily family;
         try {
             family = dataTable.getColumnFamily(getDataColumnFamilyName(indexColumnName));
               
         } catch (ColumnFamilyNotFoundException e) {
-            throw new IllegalArgumentException("Could not find column family \"" +  indexColumnName.substring(0,
pos) + "\" in index column name of \"" + indexColumnName + "\"", e);
+            return null;
         }
         try {
             return family.getPColumnForColumnName(indexColumnName.substring(pos+1));
         } catch (ColumnNotFoundException e) {
-            throw new IllegalArgumentException("Could not find column \"" +  indexColumnName.substring(pos+1)
+ "\" in index column name of \"" + indexColumnName + "\"", e);
+            return null;
         }
     }
     
@@ -686,7 +694,7 @@ public class IndexUtil {
     }
 
     public static byte[][] getViewConstants(PTable dataTable) {
-        if (dataTable.getType() != PTableType.VIEW) return null;
+        if (dataTable.getType() != PTableType.VIEW && dataTable.getType() != PTableType.PROJECTED)
return null;
         int dataPosOffset = (dataTable.getBucketNum() != null ? 1 : 0) + (dataTable.isMultiTenant()
? 1 : 0);
         ImmutableBytesWritable ptr = new ImmutableBytesWritable();
         List<byte[]> viewConstants = new ArrayList<byte[]>();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a5b5da5/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index ca4be2f..b3c7dca 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -1235,33 +1235,6 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest
{
     }
     
     @Test
-    public void testDeleteFromImmutableWithKV() throws Exception {
-        String ddl = "CREATE TABLE t (k1 VARCHAR, v1 VARCHAR, v2 VARCHAR CONSTRAINT pk PRIMARY
KEY(k1)) immutable_rows=true";
-        String indexDDL = "CREATE INDEX i ON t (v1)";
-        Connection conn = DriverManager.getConnection(getUrl());
-        try {
-            conn.createStatement().execute(ddl);
-            assertImmutableRows(conn, "T", true);
-            conn.createStatement().execute(indexDDL);
-            assertImmutableRows(conn, "I", true);
-            conn.createStatement().execute("DELETE FROM t WHERE v2 = 'foo'");
-            fail();
-        } catch (SQLException e) {
-            assertEquals(SQLExceptionCode.INVALID_FILTER_ON_IMMUTABLE_ROWS.getErrorCode(),
e.getErrorCode());
-        }
-        // Test with one index having the referenced key value column, but one not having
it.
-        // Still should fail
-        try {
-            indexDDL = "CREATE INDEX i2 ON t (v2)";
-            conn.createStatement().execute(indexDDL);
-            conn.createStatement().execute("DELETE FROM t WHERE v2 = 'foo'");
-            fail();
-        } catch (SQLException e) {
-            assertEquals(SQLExceptionCode.INVALID_FILTER_ON_IMMUTABLE_ROWS.getErrorCode(),
e.getErrorCode());
-        }
-    }
-    
-    @Test
     public void testInvalidNegativeArrayIndex() throws Exception {
         String query = "SELECT a_double_array[-20] FROM table_with_array";
         Connection conn = DriverManager.getConnection(getUrl());


Mime
View raw message