Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E932C200C37 for ; Mon, 27 Feb 2017 04:34:03 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id E7C4E160B77; Mon, 27 Feb 2017 03:34:03 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7A101160B78 for ; Mon, 27 Feb 2017 04:34:01 +0100 (CET) Received: (qmail 81669 invoked by uid 500); 27 Feb 2017 03:34:00 -0000 Mailing-List: contact commits-help@phoenix.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@phoenix.apache.org Delivered-To: mailing list commits@phoenix.apache.org Received: (qmail 81435 invoked by uid 99); 27 Feb 2017 03:34:00 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Feb 2017 03:34:00 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 12C8CDFF09; Mon, 27 Feb 2017 03:34:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: samarth@apache.org To: commits@phoenix.apache.org Date: Mon, 27 Feb 2017 03:34:00 -0000 Message-Id: <0a0373f837d845f5beb7c0f47abec8dc@git.apache.org> In-Reply-To: <45f4c13e59834004821037806ddd9414@git.apache.org> References: <45f4c13e59834004821037806ddd9414@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [02/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva) archived-at: Mon, 27 Feb 2017 03:34:04 -0000 http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index d913abd..8d48204 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -18,6 +18,8 @@ package org.apache.phoenix.util; import static org.apache.phoenix.query.QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_QUALIFIER; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -27,6 +29,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.ListIterator; import java.util.Map; import org.apache.hadoop.hbase.Cell; @@ -39,7 +42,6 @@ import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -52,6 +54,7 @@ import org.apache.hadoop.hbase.ipc.BlockingRpcCallback; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.compile.ColumnResolver; @@ -66,10 +69,12 @@ import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService; import org.apache.phoenix.coprocessor.generated.MetaDataProtos.UpdateIndexStateRequest; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; +import org.apache.phoenix.execute.MutationState.RowMutationState; import org.apache.phoenix.execute.TupleProjector; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.visitor.RowKeyExpressionVisitor; import org.apache.phoenix.hbase.index.ValueGetter; import org.apache.phoenix.hbase.index.covered.update.ColumnReference; @@ -92,9 +97,12 @@ import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.ValueSchema.Field; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PBinary; @@ -189,6 +197,11 @@ public class IndexUtil { : QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX + dataColumnFamilyName; } + public static byte[] getLocalIndexColumnFamily(byte[] dataColumnFamilyBytes) { + String dataCF = Bytes.toString(dataColumnFamilyBytes); + return getLocalIndexColumnFamily(dataCF).getBytes(); + } + public static PColumn getDataColumn(PTable dataTable, String indexColumnName) { int pos = indexColumnName.indexOf(INDEX_COLUMN_NAME_SEP); if (pos < 0) { @@ -208,7 +221,7 @@ public class IndexUtil { throw new IllegalArgumentException("Could not find column family \"" + indexColumnName.substring(0, pos) + "\" in index column name of \"" + indexColumnName + "\"", e); } try { - return family.getColumn(indexColumnName.substring(pos+1)); + return family.getPColumnForColumnName(indexColumnName.substring(pos+1)); } catch (ColumnNotFoundException e) { throw new IllegalArgumentException("Could not find column \"" + indexColumnName.substring(pos+1) + "\" in index column name of \"" + indexColumnName + "\"", e); } @@ -235,10 +248,11 @@ public class IndexUtil { private static boolean isEmptyKeyValue(PTable table, ColumnReference ref) { byte[] emptyKeyValueCF = SchemaUtil.getEmptyColumnFamily(table); + byte[] emptyKeyValueQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst(); return (Bytes.compareTo(emptyKeyValueCF, 0, emptyKeyValueCF.length, ref.getFamilyWritable() .get(), ref.getFamilyWritable().getOffset(), ref.getFamilyWritable().getLength()) == 0 && Bytes - .compareTo(QueryConstants.EMPTY_COLUMN_BYTES, 0, - QueryConstants.EMPTY_COLUMN_BYTES.length, ref.getQualifierWritable().get(), ref + .compareTo(emptyKeyValueQualifier, 0, + emptyKeyValueQualifier.length, ref.getQualifierWritable().get(), ref .getQualifierWritable().getOffset(), ref.getQualifierWritable() .getLength()) == 0); } @@ -270,10 +284,10 @@ public class IndexUtil { } public static List generateIndexData(final PTable table, PTable index, - List dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection) + final Map valuesMap, List dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection) throws SQLException { try { - final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + final ImmutableBytesPtr ptr = new ImmutableBytesPtr(); IndexMaintainer maintainer = index.getIndexMaintainer(table, connection); List indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size()); for (final Mutation dataMutation : dataMutations) { @@ -286,8 +300,6 @@ public class IndexUtil { * updating an existing row. */ if (dataMutation instanceof Put) { - // TODO: is this more efficient than looking in our mutation map - // using the key plus finding the PColumn? ValueGetter valueGetter = new ValueGetter() { @Override @@ -302,13 +314,13 @@ public class IndexUtil { if (isEmptyKeyValue(table, ref)) { return null; } - Map> familyMap = dataMutation.getFamilyCellMap(); byte[] family = ref.getFamily(); + byte[] qualifier = ref.getQualifier(); + Map> familyMap = dataMutation.getFamilyCellMap(); List kvs = familyMap.get(family); if (kvs == null) { return null; } - byte[] qualifier = ref.getQualifier(); for (Cell kv : kvs) { if (Bytes.compareTo(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), family, 0, family.length) == 0 && Bytes.compareTo(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(), qualifier, 0, qualifier.length) == 0) { @@ -442,13 +454,19 @@ public class IndexUtil { public static TupleProjector getTupleProjector(Scan scan, ColumnReference[] dataColumns) { if (dataColumns != null && dataColumns.length != 0) { KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan); - KeyValueColumnExpression[] keyValueColumns = new KeyValueColumnExpression[dataColumns.length]; + boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL) != null; + QualifierEncodingScheme encodingScheme = EncodedColumnsUtil.getQualifierEncodingScheme(scan); + Expression[] colExpressions = storeColsInSingleCell ? new SingleCellColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length]; for (int i = 0; i < dataColumns.length; i++) { - ColumnReference dataColumn = dataColumns[i]; - KeyValueColumnExpression dataColumnExpr = new KeyValueColumnExpression(keyValueSchema.getField(i), dataColumn.getFamily(), dataColumn.getQualifier()); - keyValueColumns[i] = dataColumnExpr; + byte[] family = dataColumns[i].getFamily(); + byte[] qualifier = dataColumns[i].getQualifier(); + Field field = keyValueSchema.getField(i); + Expression dataColumnExpr = + storeColsInSingleCell ? new SingleCellColumnExpression(field, family, qualifier, encodingScheme) + : new KeyValueColumnExpression(field, family, qualifier); + colExpressions[i] = dataColumnExpr; } - return new TupleProjector(keyValueSchema, keyValueColumns); + return new TupleProjector(keyValueSchema, colExpressions); } return null; } @@ -497,8 +515,13 @@ public class IndexUtil { ptr.set(indexRowKey, firstCell.getRowOffset() + offset, firstCell.getRowLength() - offset); byte[] dataRowKey = indexMaintainer.buildDataRowKey(ptr, viewConstants); Get get = new Get(dataRowKey); + ImmutableStorageScheme storageScheme = indexMaintainer.getIndexStorageScheme(); for (int i = 0; i < dataColumns.length; i++) { - get.addColumn(dataColumns[i].getFamily(), dataColumns[i].getQualifier()); + if (storageScheme == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS) { + get.addFamily(dataColumns[i].getFamily()); + } else { + get.addColumn(dataColumns[i].getFamily(), dataColumns[i].getQualifier()); + } } Result joinResult = null; if (dataRegion != null) { @@ -515,7 +538,8 @@ public class IndexUtil { if (table != null) table.close(); } } - + // at this point join result has data from the data table. We now need to take this result and + // add it to the cells that we are returning. // TODO: handle null case (but shouldn't happen) Tuple joinTuple = new ResultTuple(joinResult); // This will create a byte[] that captures all of the values from the data table @@ -523,12 +547,14 @@ public class IndexUtil { tupleProjector.getSchema().toBytes(joinTuple, tupleProjector.getExpressions(), tupleProjector.getValueBitSet(), ptr); KeyValue keyValue = - KeyValueUtil.newKeyValue(firstCell.getRowArray(),firstCell.getRowOffset(),firstCell.getRowLength(), TupleProjector.VALUE_COLUMN_FAMILY, - TupleProjector.VALUE_COLUMN_QUALIFIER, firstCell.getTimestamp(), value, 0, value.length); + KeyValueUtil.newKeyValue(firstCell.getRowArray(),firstCell.getRowOffset(),firstCell.getRowLength(), VALUE_COLUMN_FAMILY, + VALUE_COLUMN_QUALIFIER, firstCell.getTimestamp(), value, 0, value.length); result.add(keyValue); } - for (int i = 0; i < result.size(); i++) { - final Cell cell = result.get(i); + + ListIterator itr = result.listIterator(); + while (itr.hasNext()) { + final Cell cell = itr.next(); // TODO: Create DelegateCell class instead Cell newCell = new Cell() { @@ -544,7 +570,7 @@ public class IndexUtil { @Override public short getRowLength() { - return (short)(cell.getRowLength() - offset); + return (short) (cell.getRowLength() - offset); } @Override @@ -648,8 +674,7 @@ public class IndexUtil { return cell.getTagsLengthUnsigned(); } }; - // Wrap cell in cell that offsets row key - result.set(i, newCell); + itr.set(newCell); } } @@ -763,4 +788,9 @@ public class IndexUtil { } return pDataTable; } + + public static boolean isLocalIndexFamily(String family) { + return family.indexOf(LOCAL_INDEX_COLUMN_FAMILY_PREFIX) != -1; + } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java index b8850d2..071de66 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/KeyValueUtil.java @@ -25,10 +25,8 @@ import java.util.List; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index 3e10d3b..0b3849e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -648,4 +648,9 @@ public class MetaDataUtil { return Bytes.startsWith(cf, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES); } + public static final byte[] getPhysicalTableRowForView(PTable view) { + byte[] physicalTableSchemaName = Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(view.getPhysicalName().getString())); + byte[] physicalTableName = Bytes.toBytes(SchemaUtil.getTableNameFromFullName(view.getPhysicalName().getString())); + return SchemaUtil.getTableKey(ByteUtil.EMPTY_BYTE_ARRAY, physicalTableSchemaName, physicalTableName); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java index 04eb061..6af881b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java @@ -508,9 +508,9 @@ public class PhoenixRuntime { String familyName = tokens[0]; String familyColumn = tokens[1]; PColumnFamily family = table.getColumnFamily(familyName); - pColumn = family.getColumn(familyColumn); + pColumn = family.getPColumnForColumnName(familyColumn); } else { - pColumn = table.getColumn(columnName); + pColumn = table.getColumnForColumnName(columnName); } return getColumnInfo(pColumn); } @@ -1169,9 +1169,9 @@ public class PhoenixRuntime { PColumn pColumn = null; if (familyName != null) { PColumnFamily family = table.getColumnFamily(familyName); - pColumn = family.getColumn(columnName); + pColumn = family.getPColumnForColumnName(columnName); } else { - pColumn = table.getColumn(columnName); + pColumn = table.getColumnForColumnName(columnName); } return pColumn; } @@ -1209,9 +1209,9 @@ public class PhoenixRuntime { PColumn pColumn = null; if (familyName != null) { PColumnFamily family = table.getColumnFamily(familyName); - pColumn = family.getColumn(columnName); + pColumn = family.getPColumnForColumnName(columnName); } else { - pColumn = table.getColumn(columnName); + pColumn = table.getColumnForColumnName(columnName); } return pColumn; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java index dba6550..f97230b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java @@ -18,7 +18,6 @@ package org.apache.phoenix.util; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -129,63 +128,4 @@ public class ResultUtil { return Bytes.compareTo(getRawBytes(r1), getKeyOffset(r1), getKeyLength(r1), getRawBytes(r2), getKeyOffset(r2), getKeyLength(r2)); } - /** - * Binary search for latest column value without allocating memory in the process - */ - public static KeyValue getColumnLatest(Result r, byte[] family, byte[] qualifier) { - byte[] rbytes = getRawBytes(r); - int roffset = getKeyOffset(r); - int rlength = getKeyLength(r); - return getColumnLatest(r, rbytes, roffset, rlength, family, 0, family.length, qualifier, 0, qualifier.length); - } - - public static KeyValue getSearchTerm(Result r, byte[] family, byte[] qualifier) { - byte[] rbytes = getRawBytes(r); - int roffset = getKeyOffset(r); - int rlength = getKeyLength(r); - return KeyValue.createFirstOnRow(rbytes, roffset, rlength, family, 0, family.length, qualifier, 0, qualifier.length); - } - /** - * Binary search for latest column value without allocating memory in the process - */ - public static KeyValue getColumnLatest(Result r, byte[] row, int roffset, int rlength, byte[] family, int foffset, int flength, byte[] qualifier, int qoffset, int qlength) { - KeyValue searchTerm = KeyValue.createFirstOnRow(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, qlength); - return getColumnLatest(r,searchTerm); - - } - - /** - * Binary search for latest column value without allocating memory in the process - * @param r - * @param searchTerm - */ - @SuppressWarnings("deprecation") - public static KeyValue getColumnLatest(Result r, KeyValue searchTerm) { - KeyValue [] kvs = r.raw(); // side effect possibly. - if (kvs == null || kvs.length == 0) { - return null; - } - - // pos === ( -(insertion point) - 1) - int pos = Arrays.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR); - // never will exact match - if (pos < 0) { - pos = (pos+1) * -1; - // pos is now insertion point - } - if (pos == kvs.length) { - return null; // doesn't exist - } - - KeyValue kv = kvs[pos]; - if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), - searchTerm.getBuffer(), searchTerm.getFamilyOffset(), searchTerm.getFamilyLength()) != 0) { - return null; - } - if (Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), - searchTerm.getBuffer(), searchTerm.getQualifierOffset(), searchTerm.getQualifierLength()) != 0) { - return null; - } - return kv; - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index acaeb31..1fdc73b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.WritableComparator; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.ScanRanges; @@ -54,6 +55,7 @@ import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.execute.DescVarLengthFastByteComparisons; import org.apache.phoenix.filter.BooleanExpressionFilter; import org.apache.phoenix.filter.DistinctPrefixFilter; +import org.apache.phoenix.filter.MultiEncodedCQKeyValueComparisonFilter; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; @@ -63,6 +65,7 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.IllegalDataException; +import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; @@ -266,6 +269,21 @@ public class ScanUtil { scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(filter, andWithFilter))); } } + + public static void setQualifierRangesOnFilter(Scan scan, Pair minMaxQualifiers) { + Filter filter = scan.getFilter(); + if (filter != null) { + if (filter instanceof FilterList) { + for (Filter f : ((FilterList)filter).getFilters()) { + if (f instanceof MultiEncodedCQKeyValueComparisonFilter) { + ((MultiEncodedCQKeyValueComparisonFilter)f).setMinMaxQualifierRange(minMaxQualifiers); + } + } + } else if (filter instanceof MultiEncodedCQKeyValueComparisonFilter) { + ((MultiEncodedCQKeyValueComparisonFilter)filter).setMinMaxQualifierRange(minMaxQualifiers); + } + } + } public static void setTimeRange(Scan scan, long ts) { try { @@ -894,9 +912,18 @@ public class ScanUtil { } return true; } + + public static boolean hasDynamicColumns(PTable table) { + for (PColumn col : table.getColumns()) { + if (col.isDynamic()) { + return true; + } + } + return false; + } public static boolean isIndexRebuild(Scan scan) { return scan.getAttribute((BaseScannerRegionObserver.REBUILD_INDEXES)) != null; } - + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index ed90426..24e60be 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.exception.DataExceedsCapacityException; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.expression.Expression; @@ -69,6 +70,8 @@ import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.RowKeySchema; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.SaltingUtil; import org.apache.phoenix.schema.SortOrder; @@ -155,8 +158,9 @@ public class SchemaUtil { rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, column.getFamilyName().getBytes().length, column.getName().getBytes().length, valueLength); } } + byte[] emptyKeyValueKV = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst(); // Empty key value - rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, getEmptyColumnFamily(table).length, QueryConstants.EMPTY_COLUMN_BYTES.length, 0); + rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, getEmptyColumnFamily(table).length, emptyKeyValueKV.length, 0); return rowSize; } @@ -410,7 +414,7 @@ public class SchemaUtil { } } else { try { - return table.getColumnFamily(familyName.getString()).getColumn(column.getName().getString()).getName().getString(); + return table.getColumnFamily(familyName.getString()).getPColumnForColumnName(column.getName().getString()).getName().getString(); } catch (ColumnFamilyNotFoundException e) { continue; // Shouldn't happen } catch (ColumnNotFoundException e) { @@ -615,7 +619,7 @@ public class SchemaUtil { public static boolean columnExists(PTable table, String columnName) { try { - table.getColumn(columnName); + table.getColumnForColumnName(columnName); return true; } catch (ColumnNotFoundException e) { return false; @@ -1052,7 +1056,7 @@ public class SchemaUtil { } return getStrippedName(physicalTableName, indexPrefix); } - + private static String getStrippedName(String physicalTableName, String indexPrefix) { return physicalTableName.indexOf(indexPrefix) == 0 ? physicalTableName.substring(indexPrefix.length()) : physicalTableName; @@ -1073,4 +1077,21 @@ public class SchemaUtil { return normalizeIdentifier(tableName); } } + + /** + * Pads the data in ptr by the required amount for fixed width data types + */ + public static void padData(String tableName, PColumn column, ImmutableBytesWritable ptr) { + PDataType type = column.getDataType(); + byte[] byteValue = ptr.get(); + boolean isNull = type.isNull(byteValue); + Integer maxLength = column.getMaxLength(); + if (!isNull && type.isFixedWidth() && maxLength != null) { + if (ptr.getLength() < maxLength) { + type.pad(ptr, maxLength, column.getSortOrder()); + } else if (ptr.getLength() > maxLength) { + throw new DataExceedsCapacityException(tableName + "." + column.getName().getString() + " may not exceed " + maxLength + " bytes (" + type.toObject(byteValue) + ")"); + } + } + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java b/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java index ac2a850..ade5239 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/cache/TenantCacheTest.java @@ -47,7 +47,7 @@ public class TenantCacheTest { TenantCacheImpl newTenantCache = new TenantCacheImpl(memoryManager, maxServerCacheTimeToLive); ImmutableBytesPtr cacheId = new ImmutableBytesPtr(Bytes.toBytes("a")); ImmutableBytesWritable cachePtr = new ImmutableBytesWritable(Bytes.toBytes("a")); - newTenantCache.addServerCache(cacheId, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory); + newTenantCache.addServerCache(cacheId, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory, true); assertEquals(maxBytes-1, memoryManager.getAvailableMemory()); newTenantCache.removeServerCache(cacheId); assertEquals(maxBytes, memoryManager.getAvailableMemory()); @@ -63,7 +63,7 @@ public class TenantCacheTest { TenantCacheImpl cache = new TenantCacheImpl(memoryManager, maxServerCacheTimeToLive, ticker); ImmutableBytesPtr cacheId1 = new ImmutableBytesPtr(Bytes.toBytes("a")); ImmutableBytesWritable cachePtr = new ImmutableBytesWritable(Bytes.toBytes("a")); - cache.addServerCache(cacheId1, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory); + cache.addServerCache(cacheId1, cachePtr, ByteUtil.EMPTY_BYTE_ARRAY, cacheFactory, true); assertEquals(maxBytes-1, memoryManager.getAvailableMemory()); ticker.time += (maxServerCacheTimeToLive + 1) * 1000000; cache.cleanUp(); @@ -91,7 +91,7 @@ public class TenantCacheTest { } @Override - public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] txState, MemoryChunk chunk) + public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] txState, MemoryChunk chunk, boolean useProtoForIndexMaintainer) throws SQLException { return chunk; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java index 1c7477d..04b8db3 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java @@ -156,7 +156,7 @@ public class HavingCompilerTest extends BaseConnectionlessQueryTest { String query = "select count(1) from atable group by a_string having count(1) >= 1 or a_string = 'foo'"; List binds = Collections.emptyList(); Expressions expressions = compileStatement(query,binds); - PColumn aCol = ATABLE.getColumn("A_STRING"); + PColumn aCol = ATABLE.getColumnForColumnName("A_STRING"); Expression h = or( constantComparison(CompareOp.GREATER_OR_EQUAL, new CountAggregateFunction(),1L), constantComparison(CompareOp.EQUAL, http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index 2c4eccc..fa270af 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -59,6 +59,7 @@ import org.apache.phoenix.expression.aggregator.CountAggregator; import org.apache.phoenix.expression.aggregator.ServerAggregators; import org.apache.phoenix.expression.function.TimeUnit; import org.apache.phoenix.filter.ColumnProjectionFilter; +import org.apache.phoenix.filter.EncodedQualifiersColumnProjectionFilter; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; @@ -173,7 +174,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { String query = "CREATE TABLE t1 (k integer not null primary key, a.k decimal, b.k decimal)"; conn.createStatement().execute(query); PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - PColumn c = pconn.getTable(new PTableKey(pconn.getTenantId(), "T1")).getColumn("K"); + PColumn c = pconn.getTable(new PTableKey(pconn.getTenantId(), "T1")).getColumnForColumnName("K"); assertTrue(SchemaUtil.isPKColumn(c)); } finally { conn.close(); @@ -923,6 +924,25 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { } } + @Test + public void testAggregateOnColumnsNotInGroupByForImmutableEncodedTable() throws Exception { + String tableName = generateUniqueName(); + String ddl = "CREATE IMMUTABLE TABLE " + tableName + + " (a_string varchar not null, col1 integer, col2 integer" + + " CONSTRAINT pk PRIMARY KEY (a_string))"; + String query = "SELECT col1, max(a_string) from " + tableName + " group by col2"; + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.createStatement().execute(ddl); + try { + PreparedStatement statement = conn.prepareStatement(query); + statement.executeQuery(); + fail(); + } catch (SQLException e) { // expected + assertEquals(SQLExceptionCode.AGGREGATE_WITH_NOT_GROUP_BY_COLUMN.getErrorCode(), e.getErrorCode()); + } + } + } + @Test public void testRegexpSubstrSetScanKeys() throws Exception { // First test scan keys are set when the offset is 0 or 1. @@ -2322,7 +2342,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { Iterator iterator = ScanUtil.getFilterIterator(scan); while (iterator.hasNext()) { Filter filter = iterator.next(); - if (filter instanceof ColumnProjectionFilter) { + if (filter instanceof EncodedQualifiersColumnProjectionFilter) { return true; } } @@ -2624,7 +2644,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { conn.createStatement().execute(ddl); PTable table = conn.unwrap(PhoenixConnection.class).getMetaDataCache() .getTableRef(new PTableKey(null,"TABLE_WITH_DEFAULT")).getTable(); - assertNull(table.getColumn("V").getExpressionStr()); + assertNull(table.getColumnForColumnName("V").getExpressionStr()); } @Test @@ -2639,7 +2659,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { conn.createStatement().execute(ddl2); PTable table = conn.unwrap(PhoenixConnection.class).getMetaDataCache() .getTableRef(new PTableKey(null,"TABLE_WITH_DEFAULT")).getTable(); - assertNull(table.getColumn("V").getExpressionStr()); + assertNull(table.getColumnForColumnName("V").getExpressionStr()); } @Test http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java index 25280fa..6c8ac48 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryOptimizerTest.java @@ -17,8 +17,13 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.MAX_QUALIFIER; +import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.MIN_QUALIFIER; +import static org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE; +import static org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.sql.Array; @@ -32,9 +37,12 @@ import java.util.ArrayList; import java.util.List; import java.util.Properties; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseConnectionlessQueryTest; import org.apache.phoenix.query.QueryConstants; @@ -750,4 +758,47 @@ public class QueryOptimizerTest extends BaseConnectionlessQueryTest { return Joiner.on(",").join(pkColsDataTypes); } + @Test + public void testMinMaxQualifierRangeWithOrderByOnKVColumn() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + String tableName = "testMintestMinMaxQualifierRange".toUpperCase(); + conn.createStatement().execute("CREATE TABLE " + tableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 INTEGER, v2 VARCHAR) COLUMN_ENCODED_BYTES=4"); + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + ResultSet rs = stmt.executeQuery("SELECT K from " + tableName + " ORDER BY (v1)"); + assertQualifierRanges(rs, ENCODED_EMPTY_COLUMN_NAME, ENCODED_CQ_COUNTER_INITIAL_VALUE); + rs = stmt.executeQuery("SELECT K from " + tableName + " ORDER BY (v1, v2)"); + assertQualifierRanges(rs, ENCODED_EMPTY_COLUMN_NAME, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + rs = stmt.executeQuery("SELECT V2 from " + tableName + " ORDER BY (v1)"); + assertQualifierRanges(rs, ENCODED_EMPTY_COLUMN_NAME, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + rs = stmt.executeQuery("SELECT V1 from " + tableName + " ORDER BY (v1, v2)"); + assertQualifierRanges(rs, ENCODED_EMPTY_COLUMN_NAME, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + } + + @Test + public void testMinMaxQualifierRangeWithNoOrderBy() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + String tableName = "testMintestMinMaxQualifierRange".toUpperCase(); + conn.createStatement().execute("CREATE TABLE " + tableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 INTEGER, v2 VARCHAR) COLUMN_ENCODED_BYTES=4"); + PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class); + ResultSet rs = stmt.executeQuery("SELECT K from " + tableName); + assertQualifierRanges(rs, ENCODED_CQ_COUNTER_INITIAL_VALUE, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + rs = stmt.executeQuery("SELECT V2 from " + tableName); + assertQualifierRanges(rs, ENCODED_EMPTY_COLUMN_NAME, ENCODED_CQ_COUNTER_INITIAL_VALUE + 1); + rs = stmt.executeQuery("SELECT V1 from " + tableName); + assertQualifierRanges(rs, ENCODED_EMPTY_COLUMN_NAME, ENCODED_CQ_COUNTER_INITIAL_VALUE); + } + + private static void assertQualifierRanges(ResultSet rs, int minQualifier, int maxQualifier) throws SQLException { + Scan scan = rs.unwrap(PhoenixResultSet.class).getStatement().getQueryPlan().getContext().getScan(); + assertNotNull(scan.getAttribute(MIN_QUALIFIER)); + assertNotNull(scan.getAttribute(MAX_QUALIFIER)); + assertEquals(minQualifier, Bytes.toInt(scan.getAttribute(MIN_QUALIFIER))); + assertEquals(maxQualifier, Bytes.toInt(scan.getAttribute(MAX_QUALIFIER))); + } + +// private static void assertQualifierRangesNotPresent(ResultSet rs) throws SQLException { +// Scan scan = rs.unwrap(PhoenixResultSet.class).getStatement().getQueryPlan().getContext().getScan(); +// assertNull(scan.getAttribute(MIN_QUALIFIER)); +// assertNull(scan.getAttribute(MAX_QUALIFIER)); +// } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/compile/SelectStatementRewriterTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/SelectStatementRewriterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/SelectStatementRewriterTest.java index f7b6ffd..0f31c2d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/SelectStatementRewriterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/SelectStatementRewriterTest.java @@ -17,10 +17,11 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.and; import static org.apache.phoenix.util.TestUtil.constantComparison; -import static org.apache.phoenix.util.TestUtil.multiKVFilter; +import static org.apache.phoenix.util.TestUtil.multiEncodedKVFilter; import static org.apache.phoenix.util.TestUtil.singleKVFilter; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -79,14 +80,14 @@ public class SelectStatementRewriterTest extends BaseConnectionlessQueryTest { String query = "select * from atable where organization_id='" + tenantId + "' and a_integer=0 and a_string='foo'"; Filter filter = compileStatement(query); assertEquals( - multiKVFilter(and( + multiEncodedKVFilter(and( constantComparison( CompareOp.EQUAL, A_INTEGER, 0), constantComparison( CompareOp.EQUAL, A_STRING, "foo") - )), + ), TWO_BYTE_QUALIFIERS), filter); } @@ -103,14 +104,14 @@ public class SelectStatementRewriterTest extends BaseConnectionlessQueryTest { String query = "select * from atable where a_integer=0 and a_string='foo'"; Filter filter = compileStatement(query); assertEquals( - multiKVFilter(and( + multiEncodedKVFilter(and( constantComparison( CompareOp.EQUAL, A_INTEGER, 0), constantComparison( CompareOp.EQUAL, A_STRING, "foo") - )), + ), TWO_BYTE_QUALIFIERS), filter); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java index c0bff8a..8506f93 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.compile; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS; import static org.apache.phoenix.util.TestUtil.ATABLE_NAME; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.apache.phoenix.util.TestUtil.and; @@ -25,7 +26,7 @@ import static org.apache.phoenix.util.TestUtil.bindParams; import static org.apache.phoenix.util.TestUtil.columnComparison; import static org.apache.phoenix.util.TestUtil.constantComparison; import static org.apache.phoenix.util.TestUtil.in; -import static org.apache.phoenix.util.TestUtil.multiKVFilter; +import static org.apache.phoenix.util.TestUtil.multiEncodedKVFilter; import static org.apache.phoenix.util.TestUtil.not; import static org.apache.phoenix.util.TestUtil.or; import static org.apache.phoenix.util.TestUtil.singleKVFilter; @@ -52,7 +53,6 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.expression.ColumnExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.LiteralExpression; @@ -67,8 +67,10 @@ import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SaltingUtil; +import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.types.PChar; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.types.PVarchar; @@ -118,9 +120,9 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - Expression idExpression = new ColumnRef(plan.getTableRef(), plan.getTableRef().getTable().getColumn("ID").getPosition()).newColumnExpression(); + Expression idExpression = new ColumnRef(plan.getTableRef(), plan.getTableRef().getTable().getColumnForColumnName("ID").getPosition()).newColumnExpression(); Expression id = new RowKeyColumnExpression(idExpression,new RowKeyValueAccessor(plan.getTableRef().getTable().getPKColumns(),0)); - Expression company = new KeyValueColumnExpression(plan.getTableRef().getTable().getColumn("COMPANY")); + Expression company = new KeyValueColumnExpression(plan.getTableRef().getTable().getColumnForColumnName("COMPANY")); // FilterList has no equals implementation assertTrue(filter instanceof FilterList); FilterList filterList = (FilterList)filter; @@ -148,7 +150,7 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - PColumn column = plan.getTableRef().getTable().getColumn("COMPANY"); + PColumn column = plan.getTableRef().getTable().getColumnForColumnName("COMPANY"); assertEquals( singleKVFilter(constantComparison( CompareOp.EQUAL, @@ -261,10 +263,10 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); assertEquals( - multiKVFilter(columnComparison( + multiEncodedKVFilter(columnComparison( CompareOp.EQUAL, A_STRING, - B_STRING)), + B_STRING), TWO_BYTE_QUALIFIERS), filter); } @@ -296,7 +298,7 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { Filter filter = scan.getFilter(); assertEquals( - multiKVFilter(and( + multiEncodedKVFilter(and( constantComparison( CompareOp.EQUAL, A_INTEGER, @@ -304,7 +306,7 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { constantComparison( CompareOp.EQUAL, A_STRING, - "foo"))), + "foo")), TWO_BYTE_QUALIFIERS), filter); } @@ -939,17 +941,19 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - + PTable table = plan.getTableRef().getTable(); + Expression aInteger = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_INTEGER").getPosition()).newColumnExpression(); + Expression aString = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_STRING").getPosition()).newColumnExpression(); assertEquals( - multiKVFilter(and( + multiEncodedKVFilter(and( constantComparison( CompareOp.EQUAL, - A_INTEGER, + aInteger, 0), constantComparison( CompareOp.EQUAL, - A_STRING, - "foo"))), + aString, + "foo")), TWO_BYTE_QUALIFIERS), filter); byte[] startRow = PVarchar.INSTANCE.toBytes(tenantId + tenantTypeId); @@ -971,17 +975,19 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - + PTable table = plan.getTableRef().getTable(); + Expression aInteger = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_INTEGER").getPosition()).newColumnExpression(); + Expression aString = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_STRING").getPosition()).newColumnExpression(); assertEquals( - multiKVFilter(and( + multiEncodedKVFilter(and( constantComparison( CompareOp.EQUAL, - A_INTEGER, + aInteger, 0), constantComparison( CompareOp.EQUAL, - A_STRING, - "foo"))), + aString, + "foo")), TWO_BYTE_QUALIFIERS), filter); byte[] startRow = PVarchar.INSTANCE.toBytes(tenantId); http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java index 62aafa5..896fd24 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; @@ -55,12 +56,16 @@ import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.JoinTableNode.JoinType; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.parse.SelectStatement; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.EncodedCQCounter; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableRef; @@ -247,16 +252,17 @@ public class CorrelatePlanTest { for (int i = 0; i < row.length; i++) { String name = ParseNodeFactory.createTempAlias(); Expression expr = LiteralExpression.newConstant(row[i]); - columns.add(new PColumnImpl(PNameFactory.newName(name), PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), + PName colName = PNameFactory.newName(name); + columns.add(new PColumnImpl(PNameFactory.newName(name), PNameFactory.newName(VALUE_COLUMN_FAMILY), expr.getDataType(), expr.getMaxLength(), expr.getScale(), expr.isNullable(), - i, expr.getSortOrder(), null, null, false, name, false, false)); + i, expr.getSortOrder(), null, null, false, name, false, false, colName.getBytes())); } try { PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.emptyList(), false, Collections.emptyList(), null, null, false, false, false, null, - null, null, true, false, 0, 0L, Boolean.FALSE, null, false); + null, null, true, false, 0, 0L, Boolean.FALSE, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER); TableRef sourceTable = new TableRef(pTable); List sourceColumnRefs = Lists. newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java index 1b16d40..df55379 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; @@ -49,15 +50,19 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.parse.SelectStatement; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.EncodedCQCounter; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.tuple.SingleKeyValueTuple; import org.apache.phoenix.schema.tuple.Tuple; import org.junit.Test; @@ -169,15 +174,16 @@ public class LiteralResultIteratorPlanTest { for (int i = 0; i < row.length; i++) { String name = ParseNodeFactory.createTempAlias(); Expression expr = LiteralExpression.newConstant(row[i]); + PName colName = PNameFactory.newName(name); columns.add(new PColumnImpl(PNameFactory.newName(name), - PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), expr.getDataType(), expr.getMaxLength(), - expr.getScale(), expr.isNullable(), i, expr.getSortOrder(), null, null, false, name, false, false)); + PNameFactory.newName(VALUE_COLUMN_FAMILY), expr.getDataType(), expr.getMaxLength(), + expr.getScale(), expr.isNullable(), i, expr.getSortOrder(), null, null, false, name, false, false, colName.getBytes())); } try { PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections. emptyList(), false, Collections. emptyList(), null, null, false, false, - false, null, null, null, true, false, 0, 0L, false, null, false); + false, null, null, null, true, false, 0, 0L, false, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER); TableRef sourceTable = new TableRef(pTable); List sourceColumnRefs = Lists. newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/execute/MutationStateTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/MutationStateTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/MutationStateTest.java index 276d946..8553b73 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/MutationStateTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/MutationStateTest.java @@ -127,11 +127,11 @@ public class MutationStateTest { private void assertTable(String tableName1,List keyValues1,String tableName2,List keyValues2) { assertTrue("MUTATION_TEST1".equals(tableName1)); assertTrue(Bytes.equals(PUnsignedInt.INSTANCE.toBytes(111),CellUtil.cloneRow(keyValues1.get(0)))); - assertTrue("app1".equals(PVarchar.INSTANCE.toObject(CellUtil.cloneValue(keyValues1.get(0))))); + assertTrue("app1".equals(PVarchar.INSTANCE.toObject(CellUtil.cloneValue(keyValues1.get(1))))); assertTrue("MUTATION_TEST2".equals(tableName2)); assertTrue(Bytes.equals(PUnsignedInt.INSTANCE.toBytes(222),CellUtil.cloneRow(keyValues2.get(0)))); - assertTrue("app2".equals(PVarchar.INSTANCE.toObject(CellUtil.cloneValue(keyValues2.get(0))))); + assertTrue("app2".equals(PVarchar.INSTANCE.toObject(CellUtil.cloneValue(keyValues2.get(1))))); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java index 50e2721..195c2f0 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; @@ -47,6 +48,7 @@ import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.SelectStatement; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; +import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SortOrder; @@ -117,8 +119,10 @@ public class UnnestArrayPlanTest { LiteralExpression dummy = LiteralExpression.newConstant(null, arrayType); RowKeyValueAccessor accessor = new RowKeyValueAccessor(Arrays.asList(dummy), 0); UnnestArrayPlan plan = new UnnestArrayPlan(subPlan, new RowKeyColumnExpression(dummy, accessor), withOrdinality); - PColumn elemColumn = new PColumnImpl(PNameFactory.newName("ELEM"), PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), baseType, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false); - PColumn indexColumn = withOrdinality ? new PColumnImpl(PNameFactory.newName("IDX"), PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), PInteger.INSTANCE, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false) : null; + PName colName = PNameFactory.newName("ELEM"); + PColumn elemColumn = new PColumnImpl(PNameFactory.newName("ELEM"), PNameFactory.newName(VALUE_COLUMN_FAMILY), baseType, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false, colName.getBytes()); + colName = PNameFactory.newName("IDX"); + PColumn indexColumn = withOrdinality ? new PColumnImpl(colName, PNameFactory.newName(VALUE_COLUMN_FAMILY), PInteger.INSTANCE, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false, colName.getBytes()) : null; List columns = withOrdinality ? Arrays.asList(elemColumn, indexColumn) : Arrays.asList(elemColumn); ProjectedColumnExpression elemExpr = new ProjectedColumnExpression(elemColumn, columns, 0, elemColumn.getName().getString()); ProjectedColumnExpression indexExpr = withOrdinality ? new ProjectedColumnExpression(indexColumn, columns, 1, indexColumn.getName().getString()) : null; http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java index e99a71c..ba36445 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/ArrayConstructorExpressionTest.java @@ -21,8 +21,11 @@ import static org.junit.Assert.assertArrayEquals; import java.util.List; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.expression.function.ArrayElemRefExpression; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.util.ByteUtil; import org.junit.Test; @@ -31,11 +34,18 @@ import com.google.common.collect.Lists; public class ArrayConstructorExpressionTest { - private static final byte[] BYTE_ARRAY1 = new byte[]{1,2,3,4,5}; - private static final byte[] BYTE_ARRAY2 = new byte[]{6,7,8}; - + protected static final LiteralExpression CONSTANT_EXPRESSION = LiteralExpression.newConstant(QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + protected static final byte[] BYTE_ARRAY1 = new byte[]{1,2,3,4,5}; + protected static final byte[] BYTE_ARRAY2 = new byte[]{6,7,8}; + protected Expression FALSE_EVAL_EXPRESSION = new DelegateExpression(LiteralExpression.newConstant(null)) { + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + return false; + } + }; + @Test - public void testArraysWithLeadingNulls() throws Exception { + public void testLeadingNulls() throws Exception { List children = Lists.newArrayListWithExpectedSize(4); LiteralExpression nullExpression = LiteralExpression.newConstant(null); children.add(nullExpression); @@ -46,7 +56,6 @@ public class ArrayConstructorExpressionTest { ImmutableBytesPtr ptr = new ImmutableBytesPtr(); ArrayElemRefExpression arrayElemRefExpression = new ArrayElemRefExpression(Lists.newArrayList(arrayConstructorExpression)); - arrayElemRefExpression.setIndex(1); arrayElemRefExpression.evaluate(null, ptr); assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptr.copyBytesIfNecessary()); @@ -60,4 +69,5 @@ public class ArrayConstructorExpressionTest { arrayElemRefExpression.evaluate(null, ptr); assertArrayEquals(BYTE_ARRAY2, ptr.copyBytesIfNecessary()); } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java index 7ee579c..2788235 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java @@ -25,13 +25,14 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; -import org.apache.phoenix.schema.types.PBinary; -import org.apache.phoenix.schema.types.PDecimal; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; +import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; -import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PBinary; +import org.apache.phoenix.schema.types.PDecimal; +import org.apache.phoenix.schema.types.PVarchar; import org.junit.Test; public class ColumnExpressionTest { @@ -40,8 +41,9 @@ public class ColumnExpressionTest { public void testSerialization() throws Exception { int maxLen = 30; int scale = 5; - PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PDecimal.INSTANCE, maxLen, scale, - true, 20, SortOrder.getDefault(), 0, null, false, null, false, false); + PName colName = PNameFactory.newName("c1"); + PColumn column = new PColumnImpl(colName, PNameFactory.newName("f1"), PDecimal.INSTANCE, maxLen, scale, + true, 20, SortOrder.getDefault(), 0, null, false, null, false, false, colName.getBytes()); ColumnExpression colExp = new KeyValueColumnExpression(column); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); @@ -60,8 +62,9 @@ public class ColumnExpressionTest { @Test public void testSerializationWithNullScale() throws Exception { int maxLen = 30; - PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PBinary.INSTANCE, maxLen, null, - true, 20, SortOrder.getDefault(), 0, null, false, null, false, false); + PName colName = PNameFactory.newName("c1"); + PColumn column = new PColumnImpl(colName, PNameFactory.newName("f1"), PBinary.INSTANCE, maxLen, null, + true, 20, SortOrder.getDefault(), 0, null, false, null, false, false, colName.getBytes()); ColumnExpression colExp = new KeyValueColumnExpression(column); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); @@ -80,8 +83,9 @@ public class ColumnExpressionTest { @Test public void testSerializationWithNullMaxLength() throws Exception { int scale = 5; - PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PVarchar.INSTANCE, null, scale, - true, 20, SortOrder.getDefault(), 0, null, false, null, false, false); + PName colName = PNameFactory.newName("c1"); + PColumn column = new PColumnImpl(colName, PNameFactory.newName("f1"), PVarchar.INSTANCE, null, scale, + true, 20, SortOrder.getDefault(), 0, null, false, null, false, false, colName.getBytes()); ColumnExpression colExp = new KeyValueColumnExpression(column); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); @@ -99,8 +103,9 @@ public class ColumnExpressionTest { @Test public void testSerializationWithNullScaleAndMaxLength() throws Exception { - PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PDecimal.INSTANCE, null, null, true, - 20, SortOrder.getDefault(), 0, null, false, null, false, false); + PName colName = PNameFactory.newName("c1"); + PColumn column = new PColumnImpl(colName, PNameFactory.newName("f1"), PDecimal.INSTANCE, null, null, true, + 20, SortOrder.getDefault(), 0, null, false, null, false, false, colName.getBytes()); ColumnExpression colExp = new KeyValueColumnExpression(column); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java index 112109e..0d4a52f 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java @@ -109,7 +109,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest { PTable index = pconn.getTable(new PTableKey(pconn.getTenantId(),fullIndexName)); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); table.getIndexMaintainers(ptr, pconn); - List c1 = IndexMaintainer.deserialize(ptr, builder); + List c1 = IndexMaintainer.deserialize(ptr, builder, true); assertEquals(1,c1.size()); IndexMaintainer im1 = c1.get(0); @@ -135,8 +135,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest { } ValueGetter valueGetter = newValueGetter(row, valueMap); - List indexMutations = - IndexTestUtil.generateIndexData(index, table, dataMutation, ptr, builder); + List indexMutations = IndexTestUtil.generateIndexData(index, table, dataMutation, ptr, builder); assertEquals(1,indexMutations.size()); assertTrue(indexMutations.get(0) instanceof Put); Mutation indexMutation = indexMutations.get(0); @@ -311,7 +310,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest { PTable table = pconn.getTable(new PTableKey(pconn.getTenantId(), "FHA")); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); table.getIndexMaintainers(ptr, pconn); - List indexMaintainerList = IndexMaintainer.deserialize(ptr, GenericKeyValueBuilder.INSTANCE); + List indexMaintainerList = IndexMaintainer.deserialize(ptr, GenericKeyValueBuilder.INSTANCE, true); assertEquals(1,indexMaintainerList.size()); IndexMaintainer indexMaintainer = indexMaintainerList.get(0); Set indexedColumns = indexMaintainer.getIndexedColumns(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java index 791eb23..ddd8241 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java @@ -74,4 +74,4 @@ public class AggregateResultScannerTest extends BaseConnectionlessQueryTest { ResultIterator scanner = new GroupedAggregatingResultIterator(new MergeSortRowKeyResultIterator(iterators), aggregators); AssertResults.assertResults(scanner, expectedResults); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java index b74cefb..d255e0a 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java @@ -119,15 +119,15 @@ public class BaseConnectionlessQueryTest extends BaseTest { try { PTable table = conn.getTable(new PTableKey(null, ATABLE_NAME)); ATABLE = table; - ORGANIZATION_ID = new ColumnRef(new TableRef(table), table.getColumn("ORGANIZATION_ID").getPosition()).newColumnExpression(); - ENTITY_ID = new ColumnRef(new TableRef(table), table.getColumn("ENTITY_ID").getPosition()).newColumnExpression(); - A_INTEGER = new ColumnRef(new TableRef(table), table.getColumn("A_INTEGER").getPosition()).newColumnExpression(); - A_STRING = new ColumnRef(new TableRef(table), table.getColumn("A_STRING").getPosition()).newColumnExpression(); - B_STRING = new ColumnRef(new TableRef(table), table.getColumn("B_STRING").getPosition()).newColumnExpression(); - A_DATE = new ColumnRef(new TableRef(table), table.getColumn("A_DATE").getPosition()).newColumnExpression(); - A_TIME = new ColumnRef(new TableRef(table), table.getColumn("A_TIME").getPosition()).newColumnExpression(); - A_TIMESTAMP = new ColumnRef(new TableRef(table), table.getColumn("A_TIMESTAMP").getPosition()).newColumnExpression(); - X_DECIMAL = new ColumnRef(new TableRef(table), table.getColumn("X_DECIMAL").getPosition()).newColumnExpression(); + ORGANIZATION_ID = new ColumnRef(new TableRef(table), table.getColumnForColumnName("ORGANIZATION_ID").getPosition()).newColumnExpression(); + ENTITY_ID = new ColumnRef(new TableRef(table), table.getColumnForColumnName("ENTITY_ID").getPosition()).newColumnExpression(); + A_INTEGER = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_INTEGER").getPosition()).newColumnExpression(); + A_STRING = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_STRING").getPosition()).newColumnExpression(); + B_STRING = new ColumnRef(new TableRef(table), table.getColumnForColumnName("B_STRING").getPosition()).newColumnExpression(); + A_DATE = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_DATE").getPosition()).newColumnExpression(); + A_TIME = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_TIME").getPosition()).newColumnExpression(); + A_TIMESTAMP = new ColumnRef(new TableRef(table), table.getColumnForColumnName("A_TIMESTAMP").getPosition()).newColumnExpression(); + X_DECIMAL = new ColumnRef(new TableRef(table), table.getColumnForColumnName("X_DECIMAL").getPosition()).newColumnExpression(); } finally { conn.close(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index 27362ea..e444700 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -147,7 +147,8 @@ import org.apache.tephra.TransactionManager; import org.apache.tephra.TxConstants; import org.apache.tephra.distributed.TransactionService; import org.apache.tephra.metrics.TxMetricsCollector; -import org.apache.tephra.persist.InMemoryTransactionStateStorage; +import org.apache.tephra.persist.HDFSTransactionStateStorage; +import org.apache.tephra.snapshot.SnapshotCodecProvider; import org.apache.twill.discovery.DiscoveryService; import org.apache.twill.discovery.ZKDiscoveryService; import org.apache.twill.internal.utils.Networks; @@ -450,14 +451,18 @@ public abstract class BaseTest { } - protected static void setupTxManager() throws SQLException, IOException { + protected static void setTxnConfigs() throws IOException { config.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false); config.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times"); config.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1); config.setInt(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort()); config.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath()); config.setInt(TxConstants.Manager.CFG_TX_TIMEOUT, DEFAULT_TXN_TIMEOUT_SECONDS); - + config.unset(TxConstants.Manager.CFG_TX_HDFS_USER); + config.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5L); + } + + protected static void setupTxManager() throws SQLException, IOException { ConnectionInfo connInfo = ConnectionInfo.create(getUrl()); zkClient = ZKClientServices.delegate( ZKClients.reWatchOnExpire( @@ -473,7 +478,7 @@ public abstract class BaseTest { zkClient.startAndWait(); DiscoveryService discovery = new ZKDiscoveryService(zkClient); - txManager = new TransactionManager(config, new InMemoryTransactionStateStorage(), new TxMetricsCollector()); + txManager = new TransactionManager(config, new HDFSTransactionStateStorage(config, new SnapshotCodecProvider(config), new TxMetricsCollector()), new TxMetricsCollector()); txService = new TransactionService(config, zkClient, discovery, Providers.of(txManager)); txService.startAndWait(); } @@ -502,8 +507,9 @@ public abstract class BaseTest { /** * Set up the test hbase cluster. * @return url to be used by clients to connect to the cluster. + * @throws IOException */ - protected static String setUpTestCluster(@Nonnull Configuration conf, ReadOnlyProps overrideProps) { + protected static String setUpTestCluster(@Nonnull Configuration conf, ReadOnlyProps overrideProps) throws IOException { boolean isDistributedCluster = isDistributedClusterModeEnabled(conf); if (!isDistributedCluster) { return initMiniCluster(conf, overrideProps); @@ -558,8 +564,9 @@ public abstract class BaseTest { } protected static void setUpTestDriver(ReadOnlyProps serverProps, ReadOnlyProps clientProps) throws Exception { + setTxnConfigs(); String url = checkClusterInitialized(serverProps); - checkTxManagerInitialized(clientProps); + checkTxManagerInitialized(serverProps); if (driver == null) { driver = initAndRegisterTestDriver(url, clientProps); } @@ -713,26 +720,29 @@ public abstract class BaseTest { } protected static void ensureTableCreated(String url, String tableName) throws SQLException { - ensureTableCreated(url, tableName, tableName, null, null); + ensureTableCreated(url, tableName, tableName, null, null, null); } protected static void ensureTableCreated(String url, String tableName, String tableDDLType) throws SQLException { - ensureTableCreated(url, tableName, tableDDLType, null, null); + ensureTableCreated(url, tableName, tableDDLType, null, null, null); } - public static void ensureTableCreated(String url, String tableName, String tableDDLType, byte[][] splits) throws SQLException { - ensureTableCreated(url, tableName, tableDDLType, splits, null); + public static void ensureTableCreated(String url, String tableName, String tableDDLType, byte[][] splits, String tableDDLOptions) throws SQLException { + ensureTableCreated(url, tableName, tableDDLType, splits, null, tableDDLOptions); } protected static void ensureTableCreated(String url, String tableName, String tableDDLType, Long ts) throws SQLException { - ensureTableCreated(url, tableName, tableDDLType, null, ts); + ensureTableCreated(url, tableName, tableDDLType, null, ts, null); } - protected static void ensureTableCreated(String url, String tableName, String tableDDLType, byte[][] splits, Long ts) throws SQLException { + protected static void ensureTableCreated(String url, String tableName, String tableDDLType, byte[][] splits, Long ts, String tableDDLOptions) throws SQLException { String ddl = tableDDLMap.get(tableDDLType); if(!tableDDLType.equals(tableName)) { ddl = ddl.replace(tableDDLType, tableName); } + if (tableDDLOptions!=null) { + ddl += tableDDLOptions; + } createSchema(url,tableName, ts); createTestTable(url, ddl, splits, ts); } @@ -977,7 +987,7 @@ public abstract class BaseTest { } protected static void initSumDoubleValues(String tableName, byte[][] splits, String url) throws Exception { - ensureTableCreated(url, tableName, SUM_DOUBLE_NAME, splits); + ensureTableCreated(url, tableName, SUM_DOUBLE_NAME, splits, null); Properties props = new Properties(); Connection conn = DriverManager.getConnection(url, props); try { @@ -1045,26 +1055,26 @@ public abstract class BaseTest { } protected static String initATableValues(String tenantId, byte[][] splits, Date date, Long ts, String url) throws Exception { - return initATableValues(null, tenantId, splits, date, ts, url); + return initATableValues(null, tenantId, splits, date, ts, url, null); } - protected static String initATableValues(String tableName, String tenantId, byte[][] splits, Date date, Long ts, String url) throws Exception { + protected static String initATableValues(String tableName, String tenantId, byte[][] splits, Date date, Long ts, String url, String tableDDLOptions) throws Exception { if(tableName == null) { tableName = generateUniqueName(); } String tableDDLType = ATABLE_NAME; if (ts == null) { - ensureTableCreated(url, tableName, tableDDLType, splits); + ensureTableCreated(url, tableName, tableDDLType, splits, null, tableDDLOptions); } else { - ensureTableCreated(url, tableName, tableDDLType, splits, ts-5); + ensureTableCreated(url, tableName, tableDDLType, splits, ts-5, tableDDLOptions); } Properties props = new Properties(); if (ts != null) { props.setProperty(CURRENT_SCN_ATTRIB, Long.toString(ts-3)); } - Connection conn = DriverManager.getConnection(url, props); - try { + + try (Connection conn = DriverManager.getConnection(url, props)) { // Insert all rows at ts PreparedStatement stmt = conn.prepareStatement( "upsert into " + tableName + @@ -1251,12 +1261,9 @@ public abstract class BaseTest { stmt.setFloat(15, 0.09f); stmt.setDouble(16, 0.0009); stmt.execute(); - conn.commit(); - } finally { - conn.close(); - return tableName; } + return tableName; } @@ -1278,9 +1285,9 @@ public abstract class BaseTest { private static void initEntityHistoryTableValues(String tenantId, byte[][] splits, Date date, Long ts, String url) throws Exception { if (ts == null) { - ensureTableCreated(url, ENTITY_HISTORY_TABLE_NAME, ENTITY_HISTORY_TABLE_NAME, splits); + ensureTableCreated(url, ENTITY_HISTORY_TABLE_NAME, ENTITY_HISTORY_TABLE_NAME, splits, null); } else { - ensureTableCreated(url, ENTITY_HISTORY_TABLE_NAME, ENTITY_HISTORY_TABLE_NAME, splits, ts-2); + ensureTableCreated(url, ENTITY_HISTORY_TABLE_NAME, ENTITY_HISTORY_TABLE_NAME, splits, ts-2, null); } Properties props = new Properties(); @@ -1382,9 +1389,9 @@ public abstract class BaseTest { protected static void initSaltedEntityHistoryTableValues(String tenantId, byte[][] splits, Date date, Long ts, String url) throws Exception { if (ts == null) { - ensureTableCreated(url, ENTITY_HISTORY_SALTED_TABLE_NAME, ENTITY_HISTORY_SALTED_TABLE_NAME, splits); + ensureTableCreated(url, ENTITY_HISTORY_SALTED_TABLE_NAME, ENTITY_HISTORY_SALTED_TABLE_NAME, splits, null); } else { - ensureTableCreated(url, ENTITY_HISTORY_SALTED_TABLE_NAME, ENTITY_HISTORY_SALTED_TABLE_NAME, splits, ts-2); + ensureTableCreated(url, ENTITY_HISTORY_SALTED_TABLE_NAME, ENTITY_HISTORY_SALTED_TABLE_NAME, splits, ts-2, null); } Properties props = new Properties();