Mailing-List: contact dev-help@phoenix.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@phoenix.apache.org
From: snakhoda-sfdc <git@git.apache.org>
To: dev@phoenix.apache.org
Reply-To: dev@phoenix.apache.org
References: <git-pr-275-phoenix@git.apache.org>
In-Reply-To: <git-pr-275-phoenix@git.apache.org>
Subject: [GitHub] phoenix pull request #275: PHOENIX-4237: Add function to calculate Java coll...
Content-Type: text/plain
Message-Id: <20171013073858.95812DF9C3@git1-us-west.apache.org>
Date: Fri, 13 Oct 2017 07:38:57 +0000 (UTC)
archived-at: Fri, 13 Oct 2017 07:39:02 -0000

Github user snakhoda-sfdc commented on a diff in the pull request:

    https://github.com/apache/phoenix/pull/275#discussion_r144483837
  
    --- Diff: phoenix-core/src/main/java/org/apache/phoenix/expression/function/CollationKeyFunction.java ---
    @@ -0,0 +1,233 @@
    +package org.apache.phoenix.expression.function;
    +
    +import java.sql.SQLException;
    +import java.text.Collator;
    +import java.util.Arrays;
    +import java.util.List;
    +import java.util.Locale;
    +
    +import org.apache.commons.lang.BooleanUtils;
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
    +import org.apache.phoenix.expression.Expression;
    +import org.apache.phoenix.parse.FunctionParseNode;
    +import org.apache.phoenix.schema.tuple.Tuple;
    +import org.apache.phoenix.schema.types.PBoolean;
    +import org.apache.phoenix.schema.types.PDataType;
    +import org.apache.phoenix.schema.types.PInteger;
    +import org.apache.phoenix.schema.types.PIntegerArray;
    +import org.apache.phoenix.schema.types.PUnsignedIntArray;
    +import org.apache.phoenix.schema.types.PVarbinary;
    +import org.apache.phoenix.schema.types.PVarchar;
    +import org.apache.phoenix.schema.types.PhoenixArray;
    +
    +import com.force.db.i18n.LinguisticSort;
    +import com.force.i18n.LocaleUtils;
    +
    +import com.ibm.icu.impl.jdkadapter.CollatorICU;
    +import com.ibm.icu.util.ULocale;
    +
    +/**
    + * A Phoenix Function that calculates a collation key for an input string based
    + * on a caller-provided locale and collator strength and decomposition settings.
    + * 
    + * It uses the open-source grammaticus and i18n packages to obtain the collators
    + * it needs.
    + * 
    + * @author snakhoda
    + *
    + */
    +@FunctionParseNode.BuiltInFunction(name = CollationKeyFunction.NAME, args = {
    +		// input string
    +		@FunctionParseNode.Argument(allowedTypes = { PVarchar.class }),
    +		// ISO Code for Locale
    +		@FunctionParseNode.Argument(allowedTypes = { PVarchar.class }, isConstant = true),
    +		// whether to use special upper case collator
    +		@FunctionParseNode.Argument(allowedTypes = { PBoolean.class }, defaultValue = "false", isConstant = true),
    +		// collator strength
    +		@FunctionParseNode.Argument(allowedTypes = { PInteger.class }, defaultValue = "null", isConstant = true),
    +		// collator decomposition
    +		@FunctionParseNode.Argument(allowedTypes = { PInteger.class }, defaultValue = "null", isConstant = true) })
    +public class CollationKeyFunction extends ScalarFunction {
    +
    +	private static final Log LOG = LogFactory.getLog(CollationKeyFunction.class);
    +
    +	public static final String NAME = "COLLKEY";
    +
    +	public CollationKeyFunction() {
    +	}
    +
    +	public CollationKeyFunction(List<Expression> children) throws SQLException {
    +		super(children);
    +	}
    +
    +	@Override
    +	public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    +		try {
    +			String inputValue = getInputValue(tuple, ptr);
    +			String localeISOCode = getLocaleISOCode(tuple, ptr);
    +			Boolean useSpecialUpperCaseCollator = getUseSpecialUpperCaseCollator(tuple, ptr);
    +			Integer collatorStrength = getCollatorStrength(tuple, ptr);
    +			Integer collatorDecomposition = getCollatorDecomposition(tuple, ptr);
    +
    +			Locale locale = LocaleUtils.get().getLocaleByIsoCode(localeISOCode);
    +			
    +			if(LOG.isDebugEnabled()) {
    +				LOG.debug(String.format("Locale: " + locale.toLanguageTag()));
    +			}
    +			
    +			LinguisticSort linguisticSort = LinguisticSort.get(locale);
    +
    +			Collator collator = BooleanUtils.isTrue(useSpecialUpperCaseCollator)
    +					? linguisticSort.getUpperCaseCollator(false) : linguisticSort.getCollator();
    +
    +			if (collatorStrength != null) {
    +				collator.setStrength(collatorStrength);
    +			}
    +
    +			if (collatorDecomposition != null) {
    +				collator.setDecomposition(collatorDecomposition);
    +			}
    +
    +			if(LOG.isDebugEnabled()) {
    +				LOG.debug(String.format("Collator: [strength: %d, decomposition: %d], Special-Upper-Case: %s",
    +					collator.getStrength(), collator.getDecomposition(), BooleanUtils.isTrue(useSpecialUpperCaseCollator)));
    +			}
    +			
    +			byte[] collationKeyByteArray = collator.getCollationKey(inputValue).toByteArray();
    +
    +			if(LOG.isDebugEnabled()) {
    +				LOG.debug("Collation key bytes:" + Arrays.toString(collationKeyByteArray));
    +			}
    +			
    +			// byte is signed in Java, but we need unsigned values for comparison
    +			// https://www.programcreek.com/java-api-examples/index.php?api=java.text.CollationKey
    +			// Byte.toUnsignedInt will convert a byte value between [-128,127] to an int value
    +			// between [0,255]
    --- End diff --
    
    I started out with a straight byte array (VARBINARY), but realized the issue is related to 2s complement binary representation.
    
    Here's what I was able to gather:
    
    The collator's collation key produces a byte array that holds values that are supposed to be treated as unsigned. However, since byte is a signed (2s complement) datatype in Java, doing a Java comparison on them with regular operators (which is presumably what Phoenix does) produces incorrect results.
    
    For example, 00000001 represents 1 and 11111111 represents -1 in 2s complement. If we simply compare those bytes using standard operators then 11111111 is less than 00000001. But a collator key byte array that returns these two bytes would like them to be treated as "1" and "255" respectively and would need 11111111 to be greater than 00000001.
    
    Getting this effect out of a byte array basically means taking each byte and widening it (to an int) which results in the lower order bytes representing the unsigned positive number that was intended by the collation key.
    
    See this example usage of toByteArray, which does the same thing: http://icu-project.org/apiref/icu4j59m1/com/ibm/icu/text/CollationKey.html#toByteArray--
    
    BTW, Byte.toUnsignedInt in Java 8 is basically a bitwise AND with 0xFF.


---