Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 54750200C37 for ; Mon, 27 Feb 2017 07:14:38 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 530B6160B6E; Mon, 27 Feb 2017 06:14:38 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 5CBF7160B82 for ; Mon, 27 Feb 2017 07:14:36 +0100 (CET) Received: (qmail 92430 invoked by uid 500); 27 Feb 2017 06:14:35 -0000 Mailing-List: contact commits-help@phoenix.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@phoenix.apache.org Delivered-To: mailing list commits@phoenix.apache.org Received: (qmail 92091 invoked by uid 99); 27 Feb 2017 06:14:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Feb 2017 06:14:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1A2D9DFCA1; Mon, 27 Feb 2017 06:14:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: samarth@apache.org To: commits@phoenix.apache.org Date: Mon, 27 Feb 2017 06:14:41 -0000 Message-Id: <562c26ac0f984940a6fb42381719a5a9@git.apache.org> In-Reply-To: <20d3aced9317486a8bbd05b61ac63022@git.apache.org> References: <20d3aced9317486a8bbd05b61ac63022@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [08/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance archived-at: Mon, 27 Feb 2017 06:14:38 -0000 http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index fde403c..8595eda 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -368,6 +368,10 @@ public enum SQLExceptionCode { CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW(1134, "XCL34", "Altering this table property on a view is not allowed"), IMMUTABLE_TABLE_PROPERTY_INVALID(1135, "XCL35", "IMMUTABLE table property cannot be used with CREATE IMMUTABLE TABLE statement "), + + MAX_COLUMNS_EXCEEDED(1136, "XCL36", "The number of columns exceed the maximum supported by the table's qualifier encoding scheme"), + INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES(1137, "XCL37", "If IMMUTABLE_STORAGE_SCHEME property is not set to ONE_CELL_PER_COLUMN COLUMN_ENCODED_BYTES cannot be 0"), + INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE(1138, "XCL38", "IMMUTABLE_STORAGE_SCHEME property cannot be changed from/to ONE_CELL_PER_COLUMN "), /** * Implementation defined class. Phoenix internal error. (errorcode 20, sqlstate INT). http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java index 76dec2f..f6010ac 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java @@ -65,11 +65,13 @@ import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.trace.TracingIterator; import org.apache.phoenix.trace.util.Tracing; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.SQLCloseable; @@ -313,10 +315,6 @@ public abstract class BaseQueryPlan implements QueryPlan { // project is not present in the index then we need to skip this plan. if (!dataColumns.isEmpty()) { // Set data columns to be join back from data table. - serializeDataTableColumnsToJoin(scan, dataColumns); - KeyValueSchema schema = ProjectedColumnExpression.buildSchema(dataColumns); - // Set key value schema of the data columns. - serializeSchemaIntoScan(scan, schema); PTable parentTable = context.getCurrentTable().getTable(); String parentSchemaName = parentTable.getParentSchemaName().getString(); String parentTableName = parentTable.getParentTableName().getString(); @@ -327,6 +325,12 @@ public abstract class BaseQueryPlan implements QueryPlan { FACTORY.namedTable(null, TableName.create(parentSchemaName, parentTableName)), context.getConnection()).resolveTable(parentSchemaName, parentTableName); PTable dataTable = dataTableRef.getTable(); + // Set data columns to be join back from data table. + serializeDataTableColumnsToJoin(scan, dataColumns, dataTable); + KeyValueSchema schema = ProjectedColumnExpression.buildSchema(dataColumns); + // Set key value schema of the data columns. + serializeSchemaIntoScan(scan, schema); + // Set index maintainer of the local index. serializeIndexMaintainerIntoScan(scan, dataTable); // Set view constants if exists. @@ -373,7 +377,7 @@ public abstract class BaseQueryPlan implements QueryPlan { } ImmutableBytesWritable ptr = new ImmutableBytesWritable(); IndexMaintainer.serialize(dataTable, ptr, indexes, context.getConnection()); - scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD, ByteUtil.copyKeyBytesIfNecessary(ptr)); + scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO, ByteUtil.copyKeyBytesIfNecessary(ptr)); if (dataTable.isTransactional()) { scan.setAttribute(BaseScannerRegionObserver.TX_STATE, context.getConnection().getMutationState().encodeTransaction()); } @@ -429,14 +433,21 @@ public abstract class BaseQueryPlan implements QueryPlan { } } - private void serializeDataTableColumnsToJoin(Scan scan, Set dataColumns) { + private void serializeDataTableColumnsToJoin(Scan scan, Set dataColumns, PTable dataTable) { ByteArrayOutputStream stream = new ByteArrayOutputStream(); try { DataOutputStream output = new DataOutputStream(stream); + boolean storeColsInSingleCell = dataTable.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS; + if (storeColsInSingleCell) { + // if storeColsInSingleCell is true all columns of a given column family are stored in a single cell + scan.setAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL, QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + } WritableUtils.writeVInt(output, dataColumns.size()); for (PColumn column : dataColumns) { - Bytes.writeByteArray(output, column.getFamilyName().getBytes()); - Bytes.writeByteArray(output, column.getName().getBytes()); + byte[] cf = column.getFamilyName().getBytes(); + byte[] cq = column.getColumnQualifierBytes(); + Bytes.writeByteArray(output, cf); + Bytes.writeByteArray(output, cq); } scan.setAttribute(BaseScannerRegionObserver.DATA_TABLE_COLUMNS_TO_JOIN, stream.toByteArray()); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/execute/CorrelatePlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/CorrelatePlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/CorrelatePlan.java index b1d00ab..ee81c36 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/CorrelatePlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/CorrelatePlan.java @@ -159,7 +159,7 @@ public class CorrelatePlan extends DelegateQueryPlan { joined = rhsBitSet == ValueBitSet.EMPTY_VALUE_BITSET ? current : TupleProjector.mergeProjectedValue( convertLhs(current), joinedSchema, destBitSet, - rhsCurrent, rhsSchema, rhsBitSet, rhsFieldPosition); + rhsCurrent, rhsSchema, rhsBitSet, rhsFieldPosition, true); } catch (IOException e) { throw new SQLException(e); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index 4775d59..d32199b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -588,7 +588,7 @@ public class MutationState implements SQLCloseable { List indexMutations; try { indexMutations = - IndexUtil.generateIndexData(table, index, mutationsPertainingToIndex, + IndexUtil.generateIndexData(table, index, values, mutationsPertainingToIndex, connection.getKeyValueBuilder(), connection); // we may also have to include delete mutations for immutable tables if we are not processing all the tables in the mutations map if (!sendAll) { @@ -622,6 +622,7 @@ public class MutationState implements SQLCloseable { Iterator> iterator = values.entrySet().iterator(); long timestampToUse = timestamp; + Map modifiedValues = Maps.newHashMap(); while (iterator.hasNext()) { Map.Entry rowEntry = iterator.next(); byte[] onDupKeyBytes = rowEntry.getValue().getOnDupKeyBytes(); @@ -631,6 +632,10 @@ public class MutationState implements SQLCloseable { if (tableWithRowTimestampCol) { RowTimestampColInfo rowTsColInfo = state.getRowTimestampColInfo(); if (rowTsColInfo.useServerTimestamp()) { + // since we are about to modify the byte[] stored in key (which changes its hashcode) + // we need to remove the entry from the values map and add a new entry with the modified byte[] + modifiedValues.put(key, state); + iterator.remove(); // regenerate the key with this timestamp. key = getNewRowKeyWithRowTimestamp(key, timestampToUse, table); } else { @@ -671,6 +676,7 @@ public class MutationState implements SQLCloseable { if (mutationsPertainingToIndex != null) mutationsPertainingToIndex .addAll(rowMutationsPertainingToIndex); } + values.putAll(modifiedValues); } /** @@ -808,7 +814,7 @@ public class MutationState implements SQLCloseable { } for (PColumn column : columns) { if (column != null) { - resolvedTable.getColumnFamily(column.getFamilyName().getString()).getColumn(column.getName().getString()); + resolvedTable.getColumnFamily(column.getFamilyName().getString()).getPColumnForColumnName(column.getName().getString()); } } } @@ -1228,7 +1234,7 @@ public class MutationState implements SQLCloseable { } mutation.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue); if (attribValue != null) { - mutation.setAttribute(PhoenixIndexCodec.INDEX_MD, attribValue); + mutation.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, attribValue); if (txState.length > 0) { mutation.setAttribute(BaseScannerRegionObserver.TX_STATE, txState); } @@ -1523,8 +1529,8 @@ public class MutationState implements SQLCloseable { byte[] getOnDupKeyBytes() { return onDupKeyBytes; } - - Map getColumnValues() { + + public Map getColumnValues() { return columnValues; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java index f4ff289..8913f3b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/SortMergeJoinPlan.java @@ -414,7 +414,7 @@ public class SortMergeJoinPlan implements QueryPlan { return rhsBitSet == ValueBitSet.EMPTY_VALUE_BITSET ? t : TupleProjector.mergeProjectedValue( t, joinedSchema, destBitSet, - rhs, rhsSchema, rhsBitSet, rhsFieldPosition); + rhs, rhsSchema, rhsBitSet, rhsFieldPosition, true); } catch (IOException e) { throw new SQLException(e); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java index 592b68e..2126026 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/TupleProjector.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_QUALIFIER; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -51,9 +54,6 @@ import org.apache.phoenix.util.SchemaUtil; import com.google.common.base.Preconditions; public class TupleProjector { - public static final byte[] VALUE_COLUMN_FAMILY = Bytes.toBytes("_v"); - public static final byte[] VALUE_COLUMN_QUALIFIER = new byte[0]; - private static final String SCAN_PROJECTOR = "scanProjector"; private final KeyValueSchema schema; @@ -61,6 +61,8 @@ public class TupleProjector { private ValueBitSet valueSet; private final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + private static final byte[] OLD_VALUE_COLUMN_QUALIFIER = new byte[0]; + public TupleProjector(RowProjector rowProjector) { List columnProjectors = rowProjector.getColumnProjectors(); int count = columnProjectors.size(); @@ -165,11 +167,11 @@ public class TupleProjector { } public static class ProjectedValueTuple extends BaseTuple { - private ImmutableBytesWritable keyPtr = new ImmutableBytesWritable(); - private long timestamp; - private ImmutableBytesWritable projectedValue = new ImmutableBytesWritable(); - private int bitSetLen; - private KeyValue keyValue; + ImmutableBytesWritable keyPtr = new ImmutableBytesWritable(); + long timestamp; + ImmutableBytesWritable projectedValue = new ImmutableBytesWritable(); + int bitSetLen; + KeyValue keyValue; public ProjectedValueTuple(Tuple keyBase, long timestamp, byte[] projectedValue, int valueOffset, int valueLength, int bitSetLen) { keyBase.getKey(this.keyPtr); @@ -241,20 +243,63 @@ public class TupleProjector { } } + public static class OldProjectedValueTuple extends ProjectedValueTuple { + + public OldProjectedValueTuple(byte[] keyBuffer, int keyOffset, int keyLength, long timestamp, + byte[] projectedValue, int valueOffset, int valueLength, int bitSetLen) { + super(keyBuffer, keyOffset, keyLength, timestamp, projectedValue, valueOffset, valueLength, bitSetLen); + } + + public OldProjectedValueTuple(Tuple keyBase, long timestamp, byte[] projectedValue, int valueOffset, + int valueLength, int bitSetLen) { + super(keyBase, timestamp, projectedValue, valueOffset, valueLength, bitSetLen); + } + + @Override + public KeyValue getValue(int index) { + if (index != 0) { throw new IndexOutOfBoundsException(Integer.toString(index)); } + return getValue(VALUE_COLUMN_FAMILY, OLD_VALUE_COLUMN_QUALIFIER); + } + + @Override + public KeyValue getValue(byte[] family, byte[] qualifier) { + if (keyValue == null) { + keyValue = KeyValueUtil.newKeyValue(keyPtr.get(), keyPtr.getOffset(), keyPtr.getLength(), + VALUE_COLUMN_FAMILY, OLD_VALUE_COLUMN_QUALIFIER, timestamp, projectedValue.get(), + projectedValue.getOffset(), projectedValue.getLength()); + } + return keyValue; + } + + } + public ProjectedValueTuple projectResults(Tuple tuple) { byte[] bytesValue = schema.toBytes(tuple, getExpressions(), valueSet, ptr); Cell base = tuple.getValue(0); return new ProjectedValueTuple(base.getRowArray(), base.getRowOffset(), base.getRowLength(), base.getTimestamp(), bytesValue, 0, bytesValue.length, valueSet.getEstimatedLength()); } + public ProjectedValueTuple projectResults(Tuple tuple, boolean useNewValueQualifier) { + byte[] bytesValue = schema.toBytes(tuple, getExpressions(), valueSet, ptr); + Cell base = tuple.getValue(0); + if (useNewValueQualifier) { + return new ProjectedValueTuple(base.getRowArray(), base.getRowOffset(), base.getRowLength(), base.getTimestamp(), bytesValue, 0, bytesValue.length, valueSet.getEstimatedLength()); + } else { + return new OldProjectedValueTuple(base.getRowArray(), base.getRowOffset(), base.getRowLength(), base.getTimestamp(), bytesValue, 0, bytesValue.length, valueSet.getEstimatedLength()); + } + } + public static void decodeProjectedValue(Tuple tuple, ImmutableBytesWritable ptr) throws IOException { - boolean b = tuple.getValue(VALUE_COLUMN_FAMILY, VALUE_COLUMN_QUALIFIER, ptr); - if (!b) - throw new IOException("Trying to decode a non-projected value."); + boolean b = tuple.getValue(VALUE_COLUMN_FAMILY, VALUE_COLUMN_QUALIFIER, ptr); + if (!b) { + // fall back to use the old value column qualifier for backward compatibility + b = tuple.getValue(VALUE_COLUMN_FAMILY, OLD_VALUE_COLUMN_QUALIFIER, ptr); + } + if (!b) throw new IOException("Trying to decode a non-projected value."); } public static ProjectedValueTuple mergeProjectedValue(ProjectedValueTuple dest, KeyValueSchema destSchema, ValueBitSet destBitSet, - Tuple src, KeyValueSchema srcSchema, ValueBitSet srcBitSet, int offset) throws IOException { + Tuple src, KeyValueSchema srcSchema, ValueBitSet srcBitSet, int offset, boolean useNewValueColumnQualifier) throws IOException { ImmutableBytesWritable destValue = dest.getProjectedValue(); int origDestBitSetLen = dest.getBitSetLength(); destBitSet.clear(); @@ -281,7 +326,8 @@ public class TupleProjector { o = Bytes.putBytes(merged, o, srcValue.get(), srcValue.getOffset(), srcValueLen); } destBitSet.toBytes(merged, o); - return new ProjectedValueTuple(dest, dest.getTimestamp(), merged, 0, merged.length, destBitSetLen); + return useNewValueColumnQualifier ? new ProjectedValueTuple(dest, dest.getTimestamp(), merged, 0, merged.length, destBitSetLen) : + new OldProjectedValueTuple(dest, dest.getTimestamp(), merged, 0, merged.length, destBitSetLen); } public KeyValueSchema getSchema() { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java index bda1b96..51cb67e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/UnnestArrayPlan.java @@ -33,6 +33,7 @@ import org.apache.phoenix.iterate.ParallelScanGrouper; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; @@ -138,7 +139,7 @@ public class UnnestArrayPlan extends DelegateQueryPlan { @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { ptr.set(arrayPtr.get(), arrayPtr.getOffset(), arrayPtr.getLength()); - PArrayDataType.positionAtArrayElement(ptr, index++, getDataType(), getMaxLength()); + PArrayDataTypeDecoder.positionAtArrayElement(ptr, index++, getDataType(), getMaxLength()); return true; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java index c2f4dd2..8b83bf7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayConstructorExpression.java @@ -21,6 +21,7 @@ import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PArrayDataTypeEncoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.TrustedByteArrayOutputStream; @@ -31,13 +32,9 @@ import org.apache.phoenix.util.TrustedByteArrayOutputStream; public class ArrayConstructorExpression extends BaseCompoundExpression { private PDataType baseType; private int position = -1; - private int nNulls = 0; private Object[] elements; private final ImmutableBytesWritable valuePtr = new ImmutableBytesWritable(); private int estimatedSize = 0; - // store the offset postion in this. Later based on the total size move this to a byte[] - // and serialize into byte stream - private int[] offsetPos; private boolean rowKeyOrderOptimizable; public ArrayConstructorExpression() { @@ -58,9 +55,6 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { elements = new Object[getChildren().size()]; valuePtr.set(ByteUtil.EMPTY_BYTE_ARRAY); estimatedSize = PArrayDataType.estimateSize(this.children.size(), this.baseType); - if (!this.baseType.isFixedWidth()) { - offsetPos = new int[children.size()]; - } } @Override @@ -72,7 +66,6 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { public void reset() { super.reset(); position = 0; - nNulls = 0; Arrays.fill(elements, null); valuePtr.set(ByteUtil.EMPTY_BYTE_ARRAY); } @@ -85,66 +78,24 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { } TrustedByteArrayOutputStream byteStream = new TrustedByteArrayOutputStream(estimatedSize); DataOutputStream oStream = new DataOutputStream(byteStream); - try { - int noOfElements = children.size(); - nNulls = 0; - for (int i = position >= 0 ? position : 0; i < elements.length; i++) { - Expression child = children.get(i); - if (!child.evaluate(tuple, ptr)) { - if (tuple != null && !tuple.isImmutable()) { - if (position >= 0) position = i; - return false; - } - } else { - // track the offset position here from the size of the byteStream - if (!baseType.isFixedWidth()) { - // Any variable length array would follow the below order - // Every element would be seperated by a seperator byte '0' - // Null elements are counted and once a first non null element appears we - // write the count of the nulls prefixed with a seperator byte - // Trailing nulls are not taken into account - // The last non null element is followed by two seperator bytes - // For eg - // a, b, null, null, c, null would be - // 65 0 66 0 0 2 67 0 0 0 - // a null null null b c null d would be - // 65 0 0 3 66 0 67 0 0 1 68 0 0 0 - if (ptr.getLength() == 0) { - offsetPos[i] = byteStream.size(); - nNulls++; - } else { - PArrayDataType.serializeNulls(oStream, nNulls); - offsetPos[i] = byteStream.size(); - oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength()); - oStream.write(PArrayDataType.getSeparatorByte(rowKeyOrderOptimizable, getSortOrder())); - nNulls=0; - } - } else { // No nulls for fixed length - oStream.write(ptr.get(), ptr.getOffset(), ptr.getLength()); - } + PArrayDataTypeEncoder builder = + new PArrayDataTypeEncoder(byteStream, oStream, children.size(), baseType, getSortOrder(), rowKeyOrderOptimizable, PArrayDataType.SORTABLE_SERIALIZATION_VERSION); + for (int i = position >= 0 ? position : 0; i < elements.length; i++) { + Expression child = children.get(i); + if (!child.evaluate(tuple, ptr)) { + if (tuple != null && !tuple.isImmutable()) { + if (position >= 0) position = i; + return false; } - } - if (position >= 0) position = elements.length; - if (!baseType.isFixedWidth()) { - // Double seperator byte to show end of the non null array - PArrayDataType.writeEndSeperatorForVarLengthArray(oStream, getSortOrder(), rowKeyOrderOptimizable); - noOfElements = PArrayDataType.serailizeOffsetArrayIntoStream(oStream, byteStream, noOfElements, - offsetPos[offsetPos.length - 1], offsetPos); - PArrayDataType.serializeHeaderInfoIntoStream(oStream, noOfElements); - } - ptr.set(byteStream.getBuffer(), 0, byteStream.size()); - valuePtr.set(ptr.get(), ptr.getOffset(), ptr.getLength()); - return true; - } catch (IOException e) { - throw new RuntimeException("Exception while serializing the byte array"); - } finally { - try { - byteStream.close(); - oStream.close(); - } catch (IOException e) { - // Should not happen + } else { + builder.appendValue(ptr.get(), ptr.getOffset(), ptr.getLength()); } } + if (position >= 0) position = elements.length; + byte[] bytes = builder.encode(); + ptr.set(bytes, 0, bytes.length); + valuePtr.set(ptr.get(), ptr.getOffset(), ptr.getLength()); + return true; } @@ -196,4 +147,5 @@ public class ArrayConstructorExpression extends BaseCompoundExpression { buf.append(children.get(children.size()-1) + "]"); return buf.toString(); } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java index aa08adf..f5f4a29 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ExpressionType.java @@ -287,9 +287,8 @@ public enum ExpressionType { CeilYearExpression(CeilYearExpression.class), DayOfWeekFunction(DayOfWeekFunction.class), DayOfYearFunction(DayOfYearFunction.class), - DefaultValueExpression(DefaultValueExpression.class); - - + DefaultValueExpression(DefaultValueExpression.class), + ArrayColumnExpression(SingleCellColumnExpression.class); ExpressionType(Class clazz) { this.clazz = clazz; http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java index 4b5fdbb..f8432c5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/KeyValueColumnExpression.java @@ -41,33 +41,38 @@ import org.apache.phoenix.util.SchemaUtil; public class KeyValueColumnExpression extends ColumnExpression { private byte[] cf; private byte[] cq; - private String displayName; // client-side only + private String displayName; // client-side only. public KeyValueColumnExpression() { } - + public KeyValueColumnExpression(PColumn column) { - this(column, null); - } - - public KeyValueColumnExpression(PDatum column, byte[] cf, byte[] cq) { super(column); - this.cf = cf; - this.cq = cq; + this.cf = column.getFamilyName().getBytes(); + // for backward compatibility since older tables won't have columnQualifierBytes in their metadata + this.cq = column.getColumnQualifierBytes() != null ? column.getColumnQualifierBytes() : column.getName().getBytes(); + this.displayName = column.getName().getString(); } - + public KeyValueColumnExpression(PColumn column, String displayName) { super(column); this.cf = column.getFamilyName().getBytes(); - this.cq = column.getName().getBytes(); + // for backward compatibility since older tables won't have columnQualifierBytes in their metadata + this.cq = column.getColumnQualifierBytes() != null ? column.getColumnQualifierBytes() : column.getName().getBytes(); this.displayName = displayName; } + public KeyValueColumnExpression(PDatum column, byte[] cf, byte[] cq) { + super(column); + this.cf = cf; + this.cq = cq; + } + public byte[] getColumnFamily() { return cf; } - - public byte[] getColumnName() { + + public byte[] getColumnQualifier() { return cq; } @@ -120,7 +125,8 @@ public class KeyValueColumnExpression extends ColumnExpression { } @Override - public final T accept(ExpressionVisitor visitor) { + public T accept(ExpressionVisitor visitor) { return visitor.visit(this); } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java index 90882a2..f20d7e2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/LiteralExpression.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.sql.SQLException; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.IllegalDataException; @@ -214,6 +215,11 @@ public class LiteralExpression extends BaseTerminalExpression { public LiteralExpression() { } + + public LiteralExpression(byte[] byteValue) { + this.byteValue = byteValue!=null ? byteValue : ByteUtil.EMPTY_BYTE_ARRAY; + this.determinism = Determinism.ALWAYS; + } private LiteralExpression(PDataType type, Determinism determinism) { this(null, type, ByteUtil.EMPTY_BYTE_ARRAY, determinism); @@ -242,7 +248,10 @@ public class LiteralExpression extends BaseTerminalExpression { @Override public String toString() { - if (value == null) { + if (value == null && byteValue!=null) { + return Bytes.toStringBinary(byteValue); + } + else if (value == null) { return "null"; } // TODO: move into PDataType? http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java index 3a38dee..2744f35 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ProjectedColumnExpression.java @@ -154,6 +154,7 @@ public class ProjectedColumnExpression extends ColumnExpression { return Determinism.PER_INVOCATION; } + @Override public ProjectedColumnExpression clone() { return new ProjectedColumnExpression(this.column, this.columns, this.position, this.displayName); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java new file mode 100644 index 0000000..8c1e0b6 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellColumnExpression.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression; + +import static org.apache.phoenix.query.QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.compile.CreateTableCompiler.ViewWhereExpressionVisitor; +import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.ColumnValueDecoder; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PDatum; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.util.SchemaUtil; + +import com.google.common.base.Preconditions; + +/** + * + * Class to access a column that is stored in a Cell that contains all + * columns for a given column family (stored in a serialized array). + * + */ +public class SingleCellColumnExpression extends KeyValueColumnExpression { + + private int decodedColumnQualifier; + private String arrayColDisplayName; + private KeyValueColumnExpression keyValueColumnExpression; + private QualifierEncodingScheme encodingScheme; + + public SingleCellColumnExpression() { + } + + public SingleCellColumnExpression(PDatum column, byte[] cf, byte[] cq, QualifierEncodingScheme encodingScheme) { + super(column, cf, SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); + this.decodedColumnQualifier = encodingScheme.decode(cq); + this.encodingScheme = encodingScheme; + setKeyValueExpression(); + } + + public SingleCellColumnExpression(PColumn column, String displayName, QualifierEncodingScheme encodingScheme) { + super(column, column.getFamilyName().getBytes(), SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES); + Preconditions.checkNotNull(encodingScheme); + Preconditions.checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS); + this.arrayColDisplayName = displayName; + this.decodedColumnQualifier = encodingScheme.decode(column.getColumnQualifierBytes()); + this.encodingScheme = encodingScheme; + setKeyValueExpression(); + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + if (!super.evaluate(tuple, ptr)) { + return false; + } else if (ptr.getLength() == 0) { + return true; + } + // the first position is reserved and we offset maxEncodedColumnQualifier by ENCODED_CQ_COUNTER_INITIAL_VALUE (which is the minimum encoded column qualifier) + int index = decodedColumnQualifier-QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE+1; + byte serializedImmutableStorageScheme = ptr.get()[ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_BYTE]; + ImmutableStorageScheme immutableStorageScheme = ImmutableStorageScheme.fromSerializedValue(serializedImmutableStorageScheme); + // Given a ptr to the entire array, set ptr to point to a particular element within that array + ColumnValueDecoder encoderDecoder = immutableStorageScheme.getDecoder(); + return encoderDecoder.decode(ptr, index); + } + + @Override + public void readFields(DataInput input) throws IOException { + super.readFields(input); + this.decodedColumnQualifier = WritableUtils.readVInt(input); + this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)]; + setKeyValueExpression(); + } + + @Override + public void write(DataOutput output) throws IOException { + super.write(output); + WritableUtils.writeVInt(output, decodedColumnQualifier); + WritableUtils.writeVInt(output, encodingScheme.ordinal()); + } + + public KeyValueColumnExpression getKeyValueExpression() { + return keyValueColumnExpression; + } + + private void setKeyValueExpression() { + final boolean isNullable = isNullable(); + final SortOrder sortOrder = getSortOrder(); + final Integer scale = getScale(); + final Integer maxLength = getMaxLength(); + final PDataType datatype = getDataType(); + this.keyValueColumnExpression = new KeyValueColumnExpression(new PDatum() { + @Override + public boolean isNullable() { + return isNullable; + } + + @Override + public SortOrder getSortOrder() { + return sortOrder; + } + + @Override + public Integer getScale() { + return scale; + } + + @Override + public Integer getMaxLength() { + return maxLength; + } + + @Override + public PDataType getDataType() { + return datatype; + } + }, getColumnFamily(), getPositionInArray()); + } + + @Override + public String toString() { + if (arrayColDisplayName == null) { + arrayColDisplayName = SchemaUtil.getColumnDisplayName(getColumnFamily(), getColumnQualifier()); + } + return arrayColDisplayName; + } + + public byte[] getPositionInArray() { + return encodingScheme.encode(decodedColumnQualifier); + } + + @Override + public T accept(ExpressionVisitor visitor) { + //FIXME: this is ugly but can't think of a good solution. + if (visitor instanceof ViewWhereExpressionVisitor) { + return visitor.visit(this); + } else { + return super.accept(visitor); + } + } + + @Override + public boolean equals(Object obj) { + if (obj.getClass() != SingleCellColumnExpression.class) return false; + return keyValueColumnExpression.equals(((SingleCellColumnExpression)obj).getKeyValueExpression()); + } + + @Override + public int hashCode() { + return keyValueColumnExpression.hashCode(); + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java new file mode 100644 index 0000000..48485be --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/SingleCellConstructorExpression.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE + * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by + * applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.apache.phoenix.expression; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.expression.visitor.ExpressionVisitor; +import org.apache.phoenix.schema.ColumnValueEncoder; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarbinary; + +/** + * Expression used to create a single cell containing all the column values for a column family + */ +public class SingleCellConstructorExpression extends BaseCompoundExpression { + + private ImmutableStorageScheme immutableStorageScheme; + + public SingleCellConstructorExpression(ImmutableStorageScheme immutableStorageScheme, List children) { + super(children); + this.immutableStorageScheme = immutableStorageScheme; + } + + @Override + public PDataType getDataType() { + return PVarbinary.INSTANCE; + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + ColumnValueEncoder encoderDecoder = immutableStorageScheme.getEncoder(children.size()); + for (int i=0; i < children.size(); i++) { + Expression child = children.get(i); + if (!child.evaluate(tuple, ptr)) { + encoderDecoder.appendAbsentValue(); + } else { + encoderDecoder.appendValue(ptr.get(), ptr.getOffset(), ptr.getLength()); + } + } + byte[] bytes = encoderDecoder.encode(); + ptr.set(bytes, 0, bytes.length); + return true; + } + + + @Override + public void readFields(DataInput input) throws IOException { + super.readFields(input); + this.immutableStorageScheme = WritableUtils.readEnum(input, ImmutableStorageScheme.class); + } + + @Override + public void write(DataOutput output) throws IOException { + super.write(output); + WritableUtils.writeEnum(output, immutableStorageScheme); + } + + @Override + public boolean requiresFinalEvaluation() { + return true; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder("["); + if (children.size()==0) + return buf.append("]").toString(); + for (int i = 0; i < children.size() - 1; i++) { + buf.append(children.get(i) + ","); + } + buf.append(children.get(children.size()-1) + "]"); + return buf.toString(); + } + + @Override + public final T accept(ExpressionVisitor visitor) { + List l = acceptChildren(visitor, visitor.visitEnter(this)); + T t = visitor.visitLeave(this, l); + if (t == null) { + t = visitor.defaultReturn(this, l); + } + return t; + } + + public SingleCellConstructorExpression clone(List children) { + return new SingleCellConstructorExpression(immutableStorageScheme, children); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java index 6631e70..06bbced 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayElemRefExpression.java @@ -27,7 +27,7 @@ import org.apache.phoenix.expression.BaseCompoundExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.visitor.ExpressionVisitor; import org.apache.phoenix.schema.tuple.Tuple; -import org.apache.phoenix.schema.types.PArrayDataType; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PDataType; public class ArrayElemRefExpression extends BaseCompoundExpression { @@ -48,7 +48,7 @@ public class ArrayElemRefExpression extends BaseCompoundExpression { @Override public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { Expression arrayExpr = children.get(0); - return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); + return PArrayDataTypeDecoder.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java index 7a23ef5..0f3c40c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ArrayIndexFunction.java @@ -24,9 +24,9 @@ import org.apache.phoenix.expression.Expression; import org.apache.phoenix.parse.FunctionParseNode.Argument; import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; import org.apache.phoenix.parse.ParseException; +import org.apache.phoenix.schema.types.PArrayDataTypeDecoder; import org.apache.phoenix.schema.types.PBinaryArray; import org.apache.phoenix.schema.types.PInteger; -import org.apache.phoenix.schema.types.PArrayDataType; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarbinaryArray; import org.apache.phoenix.schema.SortOrder; @@ -61,7 +61,7 @@ public class ArrayIndexFunction extends ScalarFunction { throw new ParseException("Index cannot be negative :" + index); } Expression arrayExpr = children.get(0); - return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), + return PArrayDataTypeDecoder.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(), getMaxLength()); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java index af5bc2b..f2ed97b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/util/regex/JONIPattern.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.schema.SortOrder; -import org.apache.phoenix.schema.types.PArrayDataType.PArrayDataTypeBytesArrayBuilder; +import org.apache.phoenix.schema.types.PArrayDataTypeEncoder; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; @@ -159,8 +159,8 @@ public class JONIPattern extends AbstractBasePattern implements AbstractBaseSpli private boolean split(byte[] srcBytes, int srcOffset, int srcLen, ImmutableBytesWritable outPtr) { SortOrder sortOrder = SortOrder.ASC; - PArrayDataTypeBytesArrayBuilder builder = - new PArrayDataTypeBytesArrayBuilder(PVarchar.INSTANCE, sortOrder); + PArrayDataTypeEncoder builder = + new PArrayDataTypeEncoder(PVarchar.INSTANCE, sortOrder); int srcRange = srcOffset + srcLen; Matcher matcher = pattern.matcher(srcBytes, 0, srcRange); int cur = srcOffset; @@ -168,31 +168,29 @@ public class JONIPattern extends AbstractBasePattern implements AbstractBaseSpli while (true) { int nextCur = matcher.search(cur, srcRange, Option.DEFAULT); if (nextCur < 0) { - append = builder.appendElem(srcBytes, cur, srcRange - cur); - if (!append) return false; + builder.appendValue(srcBytes, cur, srcRange - cur); break; } // To handle the following case, which adds null at first. // REGEXP_SPLIT("12ONE34TWO56THREE78","[0-9]+")={null, "ONE", "TWO", "THREE", null} if (cur == matcher.getBegin()) { - builder.appendElem(srcBytes, cur, 0); + builder.appendValue(srcBytes, cur, 0); } if (cur < matcher.getBegin()) { - append = builder.appendElem(srcBytes, cur, matcher.getBegin() - cur); - if (!append) return false; + builder.appendValue(srcBytes, cur, matcher.getBegin() - cur); } cur = matcher.getEnd(); // To handle the following case, which adds null at last. // REGEXP_SPLIT("12ONE34TWO56THREE78","[0-9]+")={null, "ONE", "TWO", "THREE", null} if (cur == srcRange) { - builder.appendElem(srcBytes, cur, 0); + builder.appendValue(srcBytes, cur, 0); break; } } - byte[] bytes = builder.getBytesAndClose(SortOrder.ASC); + byte[] bytes = builder.encode(); if (bytes == null) return false; outPtr.set(bytes); return true; http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java index 8e8b32d..d79b546 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/BaseExpressionVisitor.java @@ -36,6 +36,7 @@ import org.apache.phoenix.expression.MultiplyExpression; import org.apache.phoenix.expression.NotExpression; import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -142,6 +143,11 @@ public abstract class BaseExpressionVisitor implements ExpressionVisitor { } @Override + public Iterator visitEnter(SingleCellConstructorExpression node) { + return null; + } + + @Override public Iterator visitEnter(ModulusExpression modulusExpression) { return null; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java index 00ece40..e47fb64 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneExpressionVisitor.java @@ -41,6 +41,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -80,6 +82,11 @@ public abstract class CloneExpressionVisitor extends TraverseAllExpressionVisito public Expression visit(KeyValueColumnExpression node) { return node; } + + @Override + public Expression visit(SingleCellColumnExpression node) { + return node; + } @Override public Expression visit(ProjectedColumnExpression node) { @@ -182,6 +189,11 @@ public abstract class CloneExpressionVisitor extends TraverseAllExpressionVisito public Expression visitLeave(ArrayConstructorExpression node, List l) { return isCloneNode(node, l) ? node.clone(l) : node; } + + @Override + public Expression visitLeave(SingleCellConstructorExpression node, List l) { + return isCloneNode(node, l) ? node.clone(l) : node; + } @Override public Expression visitLeave(StringConcatExpression node, List l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java index 1aeb9a9..9a56e36 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/CloneNonDeterministicExpressionVisitor.java @@ -24,6 +24,7 @@ import org.apache.phoenix.expression.Expression; public class CloneNonDeterministicExpressionVisitor extends CloneExpressionVisitor { + @Override public boolean isCloneNode(Expression node, List children) { return Determinism.PER_INVOCATION.compareTo(node.getDeterminism()) <= 0; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java index 31f340d..5936dc7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/ExpressionVisitor.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.phoenix.compile.SequenceValueExpression; import org.apache.phoenix.expression.AddExpression; import org.apache.phoenix.expression.AndExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; import org.apache.phoenix.expression.ArrayConstructorExpression; import org.apache.phoenix.expression.CaseExpression; import org.apache.phoenix.expression.CoerceExpression; @@ -42,6 +43,7 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -109,10 +111,14 @@ public interface ExpressionVisitor { public Iterator visitEnter(ArrayConstructorExpression node); public E visitLeave(ArrayConstructorExpression node, List l); + public Iterator visitEnter(SingleCellConstructorExpression node); + public E visitLeave(SingleCellConstructorExpression node, List l); + public E visit(CorrelateVariableFieldAccessExpression node); public E visit(LiteralExpression node); public E visit(RowKeyColumnExpression node); public E visit(KeyValueColumnExpression node); + public E visit(SingleCellColumnExpression node); public E visit(ProjectedColumnExpression node); public E visit(SequenceValueExpression node); http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java index 3b7067a..f5615be 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseAllExpressionVisitor.java @@ -28,7 +28,6 @@ import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; -import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.InListExpression; import org.apache.phoenix.expression.IsNullExpression; import org.apache.phoenix.expression.KeyValueColumnExpression; @@ -41,6 +40,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -121,6 +122,11 @@ public class StatelessTraverseAllExpressionVisitor extends TraverseAllExpress } @Override + public E visit(SingleCellColumnExpression node) { + return null; + } + + @Override public E visit(ProjectedColumnExpression node) { return null; } @@ -164,6 +170,11 @@ public class StatelessTraverseAllExpressionVisitor extends TraverseAllExpress public E visitLeave(ArrayConstructorExpression node, List l) { return null; } + + @Override + public E visitLeave(SingleCellConstructorExpression node, List l) { + return null; + } @Override public E visitLeave(ModulusExpression node, List l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java index 83b28bd..7f447b3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/visitor/StatelessTraverseNoExpressionVisitor.java @@ -28,7 +28,6 @@ import org.apache.phoenix.expression.CoerceExpression; import org.apache.phoenix.expression.ComparisonExpression; import org.apache.phoenix.expression.CorrelateVariableFieldAccessExpression; import org.apache.phoenix.expression.DivideExpression; -import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.InListExpression; import org.apache.phoenix.expression.IsNullExpression; import org.apache.phoenix.expression.KeyValueColumnExpression; @@ -41,6 +40,8 @@ import org.apache.phoenix.expression.OrExpression; import org.apache.phoenix.expression.ProjectedColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; import org.apache.phoenix.expression.RowValueConstructorExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.expression.SingleCellConstructorExpression; import org.apache.phoenix.expression.StringConcatExpression; import org.apache.phoenix.expression.SubtractExpression; import org.apache.phoenix.expression.function.ArrayAnyComparisonExpression; @@ -114,6 +115,11 @@ public class StatelessTraverseNoExpressionVisitor extends TraverseNoExpressio public E visit(RowKeyColumnExpression node) { return null; } + + @Override + public E visit(SingleCellColumnExpression node) { + return null; + } @Override public E visit(KeyValueColumnExpression node) { @@ -164,6 +170,11 @@ public class StatelessTraverseNoExpressionVisitor extends TraverseNoExpressio public E visitLeave(ArrayConstructorExpression node, List l) { return null; } + + @Override + public E visitLeave(SingleCellConstructorExpression node, List l) { + return null; + } @Override public E visitLeave(ModulusExpression node, List l) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java index 92e5c20..3d6843d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/ColumnProjectionFilter.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; -import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.util.EncodedColumnsUtil; /** * When selecting specific columns in a SELECT query, this filter passes only selected columns @@ -54,6 +54,8 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { private byte[] emptyCFName; private Map> columnsTracker; private Set conditionOnlyCfs; + private boolean usesEncodedColumnNames; + private byte[] emptyKVQualifier; public ColumnProjectionFilter() { @@ -61,10 +63,12 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { public ColumnProjectionFilter(byte[] emptyCFName, Map> columnsTracker, - Set conditionOnlyCfs) { + Set conditionOnlyCfs, boolean usesEncodedColumnNames) { this.emptyCFName = emptyCFName; this.columnsTracker = columnsTracker; this.conditionOnlyCfs = conditionOnlyCfs; + this.usesEncodedColumnNames = usesEncodedColumnNames; + this.emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); } @Override @@ -88,6 +92,9 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { familyMapSize--; } int conditionOnlyCfsSize = WritableUtils.readVInt(input); + usesEncodedColumnNames = conditionOnlyCfsSize > 0; + emptyKVQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(usesEncodedColumnNames).getFirst(); + conditionOnlyCfsSize = Math.abs(conditionOnlyCfsSize) - 1; // restore to the actual value. this.conditionOnlyCfs = new TreeSet(Bytes.BYTES_COMPARATOR); while (conditionOnlyCfsSize > 0) { this.conditionOnlyCfs.add(WritableUtils.readCompressedByteArray(input)); @@ -111,12 +118,13 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { } } } - // Write conditionOnlyCfs - WritableUtils.writeVInt(output, this.conditionOnlyCfs.size()); + // Encode usesEncodedColumnNames in conditionOnlyCfs size. + WritableUtils.writeVInt(output, (this.conditionOnlyCfs.size() + 1) * (usesEncodedColumnNames ? 1 : -1)); for (byte[] f : this.conditionOnlyCfs) { WritableUtils.writeCompressedByteArray(output, f); } - } + +} @Override public byte[] toByteArray() throws IOException { @@ -156,9 +164,9 @@ public class ColumnProjectionFilter extends FilterBase implements Writable { // make sure we're not holding to any of the byte[]'s ptr.set(HConstants.EMPTY_BYTE_ARRAY); if (kvs.isEmpty()) { - kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(),firstKV.getRowLength(), this.emptyCFName, - 0, this.emptyCFName.length, QueryConstants.EMPTY_COLUMN_BYTES, 0, - QueryConstants.EMPTY_COLUMN_BYTES.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0)); + kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(), + this.emptyCFName, 0, this.emptyCFName.length, emptyKVQualifier, 0, + emptyKVQualifier.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0)); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/filter/EncodedQualifiersColumnProjectionFilter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/filter/EncodedQualifiersColumnProjectionFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/filter/EncodedQualifiersColumnProjectionFilter.java new file mode 100644 index 0000000..cfacb4f --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/filter/EncodedQualifiersColumnProjectionFilter.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.filter; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_BYTES; +import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.BitSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; + +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; + +public class EncodedQualifiersColumnProjectionFilter extends FilterBase implements Writable { + + private byte[] emptyCFName; + private BitSet trackedColumns; + private QualifierEncodingScheme encodingScheme; + private Set conditionOnlyCfs; + + public EncodedQualifiersColumnProjectionFilter() {} + + public EncodedQualifiersColumnProjectionFilter(byte[] emptyCFName, BitSet trackedColumns, Set conditionCfs, QualifierEncodingScheme encodingScheme) { + checkArgument(encodingScheme != NON_ENCODED_QUALIFIERS, "Filter can only be used for encoded qualifiers"); + this.emptyCFName = emptyCFName; + this.trackedColumns = trackedColumns; + this.encodingScheme = encodingScheme; + this.conditionOnlyCfs = conditionCfs; + } + + @Override + public void readFields(DataInput input) throws IOException { + this.emptyCFName = WritableUtils.readCompressedByteArray(input); + int bitsetLongArraySize = WritableUtils.readVInt(input); + long[] bitsetLongArray = new long[bitsetLongArraySize]; + for (int i = 0; i < bitsetLongArraySize; i++) { + bitsetLongArray[i] = WritableUtils.readVLong(input); + } + this.trackedColumns = BitSet.valueOf(bitsetLongArray); + this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)]; + int conditionOnlyCfsSize = WritableUtils.readVInt(input); + this.conditionOnlyCfs = new TreeSet(Bytes.BYTES_COMPARATOR); + while (conditionOnlyCfsSize > 0) { + this.conditionOnlyCfs.add(WritableUtils.readCompressedByteArray(input)); + conditionOnlyCfsSize--; + } + } + + @Override + public void write(DataOutput output) throws IOException { + WritableUtils.writeCompressedByteArray(output, this.emptyCFName); + long[] longArrayOfBitSet = trackedColumns.toLongArray(); + WritableUtils.writeVInt(output, longArrayOfBitSet.length); + for (Long l : longArrayOfBitSet) { + WritableUtils.writeVLong(output, l); + } + WritableUtils.writeVInt(output, encodingScheme.ordinal()); + WritableUtils.writeVInt(output, this.conditionOnlyCfs.size()); + for (byte[] f : this.conditionOnlyCfs) { + WritableUtils.writeCompressedByteArray(output, f); + } + } + + @Override + public byte[] toByteArray() throws IOException { + return Writables.getBytes(this); + } + + public static EncodedQualifiersColumnProjectionFilter parseFrom(final byte [] pbBytes) throws DeserializationException { + try { + return (EncodedQualifiersColumnProjectionFilter)Writables.getWritable(pbBytes, new EncodedQualifiersColumnProjectionFilter()); + } catch (IOException e) { + throw new DeserializationException(e); + } + } + + @Override + public void filterRowCells(List kvs) throws IOException { + if (kvs.isEmpty()) return; + Cell firstKV = kvs.get(0); + Iterables.removeIf(kvs, new Predicate() { + @Override + public boolean apply(Cell kv) { + int qualifier = encodingScheme.decode(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()); + return !trackedColumns.get(qualifier); + } + }); + if (kvs.isEmpty()) { + kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(), + this.emptyCFName, 0, this.emptyCFName.length, ENCODED_EMPTY_COLUMN_BYTES, 0, + ENCODED_EMPTY_COLUMN_BYTES.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0)); + } + } + + @Override + public boolean hasFilterRow() { + return true; + } + + @Override + public boolean isFamilyEssential(byte[] name) { + return conditionOnlyCfs.isEmpty() || this.conditionOnlyCfs.contains(name); + } + + @Override + public String toString() { + return ""; + } + + @Override + public ReturnCode filterKeyValue(Cell ignored) throws IOException { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + + interface ColumnTracker { + + } +}