phoenix-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From JamesRTaylor <...@git.apache.org>
Subject [GitHub] phoenix pull request #275: PHOENIX-4237: Add function to calculate Java coll...
Date Fri, 13 Oct 2017 18:01:45 GMT
Github user JamesRTaylor commented on a diff in the pull request:

    https://github.com/apache/phoenix/pull/275#discussion_r144620511
  
    --- Diff: phoenix-core/src/main/java/org/apache/phoenix/expression/function/CollationKeyFunction.java
---
    @@ -0,0 +1,233 @@
    +package org.apache.phoenix.expression.function;
    +
    +import java.sql.SQLException;
    +import java.text.Collator;
    +import java.util.Arrays;
    +import java.util.List;
    +import java.util.Locale;
    +
    +import org.apache.commons.lang.BooleanUtils;
    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
    +import org.apache.phoenix.expression.Expression;
    +import org.apache.phoenix.parse.FunctionParseNode;
    +import org.apache.phoenix.schema.tuple.Tuple;
    +import org.apache.phoenix.schema.types.PBoolean;
    +import org.apache.phoenix.schema.types.PDataType;
    +import org.apache.phoenix.schema.types.PInteger;
    +import org.apache.phoenix.schema.types.PIntegerArray;
    +import org.apache.phoenix.schema.types.PUnsignedIntArray;
    +import org.apache.phoenix.schema.types.PVarbinary;
    +import org.apache.phoenix.schema.types.PVarchar;
    +import org.apache.phoenix.schema.types.PhoenixArray;
    +
    +import com.force.db.i18n.LinguisticSort;
    +import com.force.i18n.LocaleUtils;
    +
    +import com.ibm.icu.impl.jdkadapter.CollatorICU;
    +import com.ibm.icu.util.ULocale;
    +
    +/**
    + * A Phoenix Function that calculates a collation key for an input string based
    + * on a caller-provided locale and collator strength and decomposition settings.
    + * 
    + * It uses the open-source grammaticus and i18n packages to obtain the collators
    + * it needs.
    + * 
    + * @author snakhoda
    + *
    + */
    +@FunctionParseNode.BuiltInFunction(name = CollationKeyFunction.NAME, args = {
    +		// input string
    +		@FunctionParseNode.Argument(allowedTypes = { PVarchar.class }),
    +		// ISO Code for Locale
    +		@FunctionParseNode.Argument(allowedTypes = { PVarchar.class }, isConstant = true),
    +		// whether to use special upper case collator
    +		@FunctionParseNode.Argument(allowedTypes = { PBoolean.class }, defaultValue = "false",
isConstant = true),
    +		// collator strength
    +		@FunctionParseNode.Argument(allowedTypes = { PInteger.class }, defaultValue = "null",
isConstant = true),
    +		// collator decomposition
    +		@FunctionParseNode.Argument(allowedTypes = { PInteger.class }, defaultValue = "null",
isConstant = true) })
    +public class CollationKeyFunction extends ScalarFunction {
    +
    +	private static final Log LOG = LogFactory.getLog(CollationKeyFunction.class);
    +
    +	public static final String NAME = "COLLKEY";
    +
    +	public CollationKeyFunction() {
    +	}
    +
    +	public CollationKeyFunction(List<Expression> children) throws SQLException {
    +		super(children);
    +	}
    +
    +	@Override
    +	public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    +		try {
    +			String inputValue = getInputValue(tuple, ptr);
    +			String localeISOCode = getLocaleISOCode(tuple, ptr);
    +			Boolean useSpecialUpperCaseCollator = getUseSpecialUpperCaseCollator(tuple, ptr);
    +			Integer collatorStrength = getCollatorStrength(tuple, ptr);
    +			Integer collatorDecomposition = getCollatorDecomposition(tuple, ptr);
    +
    +			Locale locale = LocaleUtils.get().getLocaleByIsoCode(localeISOCode);
    +			
    +			if(LOG.isDebugEnabled()) {
    +				LOG.debug(String.format("Locale: " + locale.toLanguageTag()));
    +			}
    +			
    +			LinguisticSort linguisticSort = LinguisticSort.get(locale);
    +
    +			Collator collator = BooleanUtils.isTrue(useSpecialUpperCaseCollator)
    +					? linguisticSort.getUpperCaseCollator(false) : linguisticSort.getCollator();
    +
    +			if (collatorStrength != null) {
    +				collator.setStrength(collatorStrength);
    +			}
    +
    +			if (collatorDecomposition != null) {
    +				collator.setDecomposition(collatorDecomposition);
    +			}
    +
    +			if(LOG.isDebugEnabled()) {
    +				LOG.debug(String.format("Collator: [strength: %d, decomposition: %d], Special-Upper-Case:
%s",
    +					collator.getStrength(), collator.getDecomposition(), BooleanUtils.isTrue(useSpecialUpperCaseCollator)));
    +			}
    +			
    +			byte[] collationKeyByteArray = collator.getCollationKey(inputValue).toByteArray();
    +
    +			if(LOG.isDebugEnabled()) {
    +				LOG.debug("Collation key bytes:" + Arrays.toString(collationKeyByteArray));
    +			}
    +			
    +			// byte is signed in Java, but we need unsigned values for comparison
    +			// https://www.programcreek.com/java-api-examples/index.php?api=java.text.CollationKey
    +			// Byte.toUnsignedInt will convert a byte value between [-128,127] to an int value
    +			// between [0,255]
    --- End diff --
    
    Let's start with the tests. That conversion should not be necessary.


---

Mime
View raw message