Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8037611B28 for ; Wed, 16 Apr 2014 06:33:22 +0000 (UTC) Received: (qmail 92782 invoked by uid 500); 16 Apr 2014 06:33:22 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 92723 invoked by uid 500); 16 Apr 2014 06:33:18 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 92484 invoked by uid 99); 16 Apr 2014 06:33:16 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Apr 2014 06:33:16 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 33CF2930E22; Wed, 16 Apr 2014 06:33:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hyunsik@apache.org To: commits@tajo.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: git commit: TAJO-746: Implements function COALESCE. (hyoungjunkim via hyunsik) Date: Wed, 16 Apr 2014 06:33:16 +0000 (UTC) Repository: tajo Updated Branches: refs/heads/master 1d24a25ac -> 014f9f3ef TAJO-746: Implements function COALESCE. (hyoungjunkim via hyunsik) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/014f9f3e Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/014f9f3e Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/014f9f3e Branch: refs/heads/master Commit: 014f9f3ef32b944f901ec97e60f1cec0a7cb2ac1 Parents: 1d24a25 Author: Hyunsik Choi Authored: Wed Apr 16 15:31:23 2014 +0900 Committer: Hyunsik Choi Committed: Wed Apr 16 15:31:23 2014 +0900 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../org/apache/tajo/catalog/CatalogUtil.java | 286 ++++++++++++++++++- .../apache/tajo/catalog/TestCatalogUtil.java | 34 ++- .../org/apache/tajo/catalog/CatalogServer.java | 105 +------ .../org/apache/tajo/catalog/TestCatalog.java | 4 +- .../org/apache/tajo/client/ResultSetUtil.java | 6 +- .../org/apache/tajo/jdbc/TajoResultSetBase.java | 1 - tajo-common/src/main/proto/DataTypes.proto | 2 - .../tajo/engine/function/builtin/Coalesce.java | 44 +++ .../engine/function/builtin/CoalesceDouble.java | 43 +++ .../engine/function/builtin/CoalesceLong.java | 44 +++ .../engine/function/builtin/CoalesceString.java | 44 +++ .../tajo/engine/parser/HiveQLAnalyzer.java | 2 +- .../apache/tajo/engine/parser/SQLAnalyzer.java | 5 +- .../tajo/engine/planner/ExprsVerifier.java | 2 +- .../function/TestConditionalExpressions.java | 82 ++++++ 16 files changed, 594 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index c8ca4c4..a643f1a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,8 @@ Release 0.8.0 - unreleased NEW FEATURES + TAJO-746: Implements function COALESCE. (hyoungjunkim via hyunsik) + TAJO-616: SequenceFile support. (jaehwa) TAJO-480: Umbrella Jira for adding ALTER TABLE statement. http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 7568b26..13eed38 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -25,7 +25,9 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.catalog.proto.CatalogProtos.TableDescProto; +import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.datum.exception.InvalidOperationException; import org.apache.tajo.util.StringUtils; import java.sql.Connection; @@ -34,6 +36,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; @@ -370,6 +373,286 @@ public class CatalogUtil { return sb.toString(); } + public static boolean isArrayType(TajoDataTypes.Type type) { + return type.toString().endsWith("_ARRAY"); + } + + /** + * Checking if the given parameter types are compatible to the defined types of the function. + * It also considers variable-length function invocations. + * + * @param definedTypes The defined function types + * @param givenTypes The given parameter types + * @return True if the parameter types are compatible to the defined types. + */ + public static boolean isMatchedFunction(List definedTypes, List givenTypes) { + + // below check handles the following cases: + // + // defined arguments given params RESULT + // () () T + // () (arg1) F + + if (definedTypes == null || definedTypes.isEmpty()) { // if no defined argument + if (givenTypes == null || givenTypes.isEmpty()) { + return true; // if no defined argument as well as no given parameter + } else { + return false; // if no defined argument but there is one or more given parameters. + } + } + + // if the number of the given parameters are less than, the invocation is invalid. + // It should already return false. + // + // below check handles the following example cases: + // + // defined given arguments + // (a) () + // (a,b) (a) + + int definedSize = definedTypes == null ? 0 : definedTypes.size(); + int givenParamSize = givenTypes == null ? 0 : givenTypes.size(); + int paramDiff = givenParamSize - definedSize; + if (paramDiff < 0) { + return false; + } + + // if the lengths of both defined and given parameter types are the same to each other + if (paramDiff == 0) { + return checkIfMatchedNonVarLengthFunction(definedTypes, givenTypes, definedSize, true); + + } else { // if variable length parameter match is suspected + + // Invocation parameters can be divided into two parts: non-variable part and variable part. + // + // For example, the function definition is as follows: + // + // c = func(a int, b float, c text[]) + // + // If the function invocation is as follows. We can divided them into two parts as we mentioned above. + // + // func( 1, 3.0, 'b', 'c', 'd', ...) + // <------------> <-----------------------> + // non-variable part variable part + + // check if the candidate function is a variable-length function. + if (!checkIfVariableLengthParamDefinition(definedTypes)) { + return false; + } + + // check only non-variable part + if (!checkIfMatchedNonVarLengthFunction(definedTypes, givenTypes, definedSize - 1, false)) { + return false; + } + + //////////////////////////////////////////////////////////////////////////////// + // The below code is for checking the variable part of a candidate function. + //////////////////////////////////////////////////////////////////////////////// + + // Get a primitive type of the last defined parameter (array) + TajoDataTypes.Type primitiveTypeOfLastDefinedParam = + getPrimitiveTypeOf(definedTypes.get(definedTypes.size() - 1).getType()); + + Type basisTypeOfVarLengthType = null; + Type [] varLengthTypesOfGivenParams = new Type[paramDiff + 1]; // including last parameter + for (int i = 0,j = (definedSize - 1); j < givenParamSize; i++, j++) { + varLengthTypesOfGivenParams[i] = givenTypes.get(j).getType(); + + // chooses one basis type for checking the variable part + if (givenTypes.get(j).getType() != Type.NULL_TYPE) { + basisTypeOfVarLengthType = givenTypes.get(j).getType(); + } + } + + // If basis type is null, it means that all params in variable part is NULL_TYPE. + // In this case, we set NULL_TYPE to the basis type. + if (basisTypeOfVarLengthType == null) { + basisTypeOfVarLengthType = Type.NULL_TYPE; + } + + // Check if a basis param type is compatible to the variable parameter in the function definition. + if (!(primitiveTypeOfLastDefinedParam == basisTypeOfVarLengthType || + isCompatibleType(primitiveTypeOfLastDefinedParam, basisTypeOfVarLengthType))) { + return false; + } + + // If all parameters are equivalent to the basis type + for (TajoDataTypes.Type type : varLengthTypesOfGivenParams) { + if (type != Type.NULL_TYPE && type != basisTypeOfVarLengthType) { + return false; + } + } + + return true; + } + } + + /** + * It is used when the function definition and function invocation whose the number of parameters are the same. + * + * @param definedTypes The parameter definition of a function + * @param givenTypes invoked parameters + * @param n Should how many types be checked? + * @param lastArrayCompatible variable-length compatibility is allowed if true. + * @return True the parameter definition and invoked definition are compatible to each other + */ + public static boolean checkIfMatchedNonVarLengthFunction(List definedTypes, List givenTypes, + int n, boolean lastArrayCompatible) { + for (int i = 0; i < n; i++) { + Type definedType = definedTypes.get(i).getType(); + Type givenType = givenTypes.get(i).getType(); + + if (lastArrayCompatible) { + if (!CatalogUtil.checkIfCompatibleIncludingArray(definedType, givenType)) { + return false; + } + } else { + if (!CatalogUtil.isCompatibleType(definedType, givenType)) { + return false; + } + } + } + + return true; + } + + /** + * Check if both are compatible to each other. This function includes + * the compatibility between primitive type and array type. + * For example, INT8 and INT8_ARRAY is compatible. + * This method is used to find variable-length functions. + * + * @param defined One parameter of the function definition + * @param given One parameter of the invoked parameters + * @return True if compatible. + */ + public static boolean checkIfCompatibleIncludingArray(Type defined, Type given) { + boolean compatible = isCompatibleType(defined, given); + + if (compatible) { + return true; + } + + if (isArrayType(defined)) { + TajoDataTypes.Type primitiveTypeOfDefined = getPrimitiveTypeOf(defined); + return isCompatibleType(primitiveTypeOfDefined, given); + } else { + return false; + } + } + + /** + * Check if the parameter definition can be variable length. + * + * @param definedTypes The parameter definition of a function. + * @return True if the parameter definition can be variable length. + */ + public static boolean checkIfVariableLengthParamDefinition(List definedTypes) { + // Get the last param type of the function definition. + Type lastDefinedParamType = definedTypes.get(definedTypes.size() - 1).getType(); + + // Check if this function is variable function. + // if the last defined parameter is not array, it is not a variable length function. It will be false. + return CatalogUtil.isArrayType(lastDefinedParamType); + } + + public static Type getPrimitiveTypeOf(Type arrayType) { + switch (arrayType) { + case BOOLEAN_ARRAY: return Type.BOOLEAN; + case UINT1_ARRAY: return Type.UINT1; + case UINT2_ARRAY: return Type.UINT2; + case UINT4_ARRAY: return Type.UINT4; + case UINT8_ARRAY: return Type.UINT8; + case INT1_ARRAY: return Type.INT1; + case INT2_ARRAY: return Type.INT2; + case INT4_ARRAY: return Type.INT4; + case INT8_ARRAY: return Type.INT8; + case FLOAT4_ARRAY: return Type.FLOAT4; + case FLOAT8_ARRAY: return Type.FLOAT8; + case NUMERIC_ARRAY: return Type.NUMERIC; + case CHAR_ARRAY: return Type.CHAR; + case NCHAR_ARRAY: return Type.NCHAR; + case VARCHAR_ARRAY: return Type.VARCHAR; + case NVARCHAR_ARRAY: return Type.NVARCHAR; + case TEXT_ARRAY: return Type.TEXT; + case DATE_ARRAY: return Type.DATE; + case TIME_ARRAY: return Type.TIME; + case TIMEZ_ARRAY: return Type.TIMEZ; + case TIMESTAMP_ARRAY: return Type.TIMESTAMP; + case TIMESTAMPZ_ARRAY: return Type.TIMESTAMPZ; + case INTERVAL_ARRAY: return Type.INTERVAL; + default: throw new InvalidOperationException("Invalid array type: " + arrayType.name()); + } + } + + public static boolean isCompatibleType(final Type definedType, final Type givenType) { + boolean flag = false; + if (givenType == Type.NULL_TYPE) { + flag = true; + } else if (definedType == Type.ANY) { + flag = true; + } else if (givenType.getNumber() > definedType.getNumber()) { + // NO POINT IN GOING FORWARD BECAUSE THE DATA TYPE CANNOT BE UPPER CASTED + flag = false; + } else { + //argType.getNumber() < exitingType.getNumber() + int exitingTypeNumber = definedType.getNumber(); + int argTypeNumber = givenType.getNumber(); + + if (Type.INT1.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.INT8.getNumber()) { + // INT1 ==> INT2 ==> INT4 ==> INT8 + if (Type.INT1.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.UINT1.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.UINT8.getNumber()) { + // UINT1 ==> UINT2 ==> UINT4 ==> UINT8 + if (Type.UINT1.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.FLOAT4.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.NUMERIC.getNumber()) { + // FLOAT4 ==> FLOAT8 ==> NUMERIC ==> DECIMAL + if (Type.FLOAT4.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.CHAR.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.TEXT.getNumber()) { + // CHAR ==> NCHAR ==> VARCHAR ==> NVARCHAR ==> TEXT + if (Type.CHAR.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.BIT.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.VARBINARY.getNumber()) { + // BIT ==> VARBIT ==> BINARY ==> VARBINARY + if (Type.BIT.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.INT1_ARRAY.getNumber() <= exitingTypeNumber + && exitingTypeNumber <= Type.INT8_ARRAY.getNumber()) { + // INT1_ARRAY ==> INT2_ARRAY ==> INT4_ARRAY ==> INT8_ARRAY + if (Type.INT1_ARRAY.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.UINT1_ARRAY.getNumber() <= exitingTypeNumber + && exitingTypeNumber <= Type.UINT8_ARRAY.getNumber()) { + // UINT1_ARRAY ==> UINT2_ARRAY ==> UINT4_ARRAY ==> UINT8_ARRAY + if (Type.UINT1_ARRAY.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.FLOAT4_ARRAY.getNumber() <= exitingTypeNumber + && exitingTypeNumber <= Type.FLOAT8_ARRAY.getNumber()) { + // FLOAT4_ARRAY ==> FLOAT8_ARRAY ==> NUMERIC_ARRAY ==> DECIMAL_ARRAY + if (Type.FLOAT4_ARRAY.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } else if (Type.CHAR_ARRAY.getNumber() <= exitingTypeNumber + && exitingTypeNumber <= Type.TEXT_ARRAY.getNumber()) { + // CHAR_ARRAY ==> NCHAR_ARRAY ==> VARCHAR_ARRAY ==> NVARCHAR_ARRAY ==> TEXT_ARRAY + if (Type.TEXT_ARRAY.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { + flag = true; + } + } + } + return flag; + } + public static CatalogProtos.TableIdentifierProto buildTableIdentifier(String databaseName, String tableName) { CatalogProtos.TableIdentifierProto.Builder builder = CatalogProtos.TableIdentifierProto.newBuilder(); builder.setDatabaseName(databaseName); @@ -438,7 +721,8 @@ public class CatalogUtil { } } - public static AlterTableDesc renameColumn(String tableName, String oldColumName, String newColumName, AlterTableType alterTableType) { + public static AlterTableDesc renameColumn(String tableName, String oldColumName, String newColumName, + AlterTableType alterTableType) { final AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setTableName(tableName); alterTableDesc.setColumnName(oldColumName); http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java index 75149e9..3c0536b 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java @@ -18,10 +18,12 @@ package org.apache.tajo.catalog; -import org.junit.Test; import org.apache.tajo.common.TajoDataTypes.Type; +import org.junit.Test; -import static org.junit.Assert.assertEquals; +import java.util.Arrays; + +import static org.junit.Assert.*; public class TestCatalogUtil { @Test @@ -51,4 +53,32 @@ public class TestCatalogUtil { assertEquals(normalized[i], CatalogUtil.normalizeIdentifier(sources[i])); } } + + @Test + public final void testIsCompatibleType() { + assertFalse(CatalogUtil.isCompatibleType(Type.INT4, Type.INT8)); + assertTrue(CatalogUtil.isCompatibleType(Type.INT8, Type.INT4)); + assertFalse(CatalogUtil.isCompatibleType(Type.FLOAT4, Type.FLOAT8)); + assertTrue(CatalogUtil.isCompatibleType(Type.FLOAT8, Type.FLOAT4)); + + assertFalse(CatalogUtil.isCompatibleType(Type.FLOAT8, Type.INT4)); + + assertFalse(CatalogUtil.isCompatibleType(Type.FLOAT8_ARRAY, Type.TEXT_ARRAY)); + assertFalse(CatalogUtil.isCompatibleType(Type.TEXT_ARRAY, Type.FLOAT8_ARRAY)); + } + + @Test + public final void testCompareDataTypeIncludeVariableLength() { + assertTrue(CatalogUtil.isMatchedFunction( + Arrays.asList(CatalogUtil.newSimpleDataTypeArray(Type.FLOAT8, Type.INT4)), Arrays.asList(CatalogUtil.newSimpleDataTypeArray(Type.FLOAT4, Type.INT4)) + )); + + assertFalse(CatalogUtil.isMatchedFunction( + Arrays.asList(CatalogUtil.newSimpleDataTypeArray(Type.FLOAT4, Type.INT4)), Arrays.asList(CatalogUtil.newSimpleDataTypeArray(Type.FLOAT8, Type.INT4)) + )); + + assertTrue(CatalogUtil.isMatchedFunction( + Arrays.asList(CatalogUtil.newSimpleDataTypeArray(Type.FLOAT8, Type.INT8_ARRAY)), Arrays.asList(CatalogUtil.newSimpleDataTypeArray(Type.FLOAT4, Type.INT4, Type.INT4)) + )); + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index b21bdf0..c0f6d36 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -30,7 +30,6 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.annotation.ThreadSafe; import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService; import org.apache.tajo.catalog.exception.*; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.store.CatalogStore; import org.apache.tajo.catalog.store.DerbyStore; @@ -39,7 +38,6 @@ import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.rpc.BlockingRpcServer; -import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto; @@ -58,7 +56,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import static org.apache.tajo.catalog.proto.CatalogProtos.FunctionType.*; -import static org.apache.tajo.common.TajoDataTypes.Type; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListProto; /** @@ -842,7 +839,7 @@ public class CatalogServer extends AbstractService { } private boolean containFunction(String signature, FunctionType type, List params) { - return findFunction(signature, type, params,false) != null; + return findFunction(signature, type, params, false) != null; } private List findFunction(String signature) { @@ -856,8 +853,9 @@ public class CatalogServer extends AbstractService { return existing; } } + } - /* + /* * * FALL BACK to look for nearest match * WORKING BUT BAD WAY TO IMPLEMENT.I WOULD RATHER implement compareTo in FunctionDesc to keep them @@ -866,10 +864,10 @@ public class CatalogServer extends AbstractService { * to implement compareTo so decided to take the shortcut. * * */ - for (FunctionDescProto existing : functions.get(signature)) { - if (existing.getParameterTypesList() != null && compareDataType(params, existing.getParameterTypesList())) { - return existing; - } + for (FunctionDescProto existing : functions.get(signature)) { + if (existing.getParameterTypesList() != null && + CatalogUtil.isMatchedFunction(existing.getParameterTypesList(), params)) { + return existing; } } return null; @@ -878,7 +876,6 @@ public class CatalogServer extends AbstractService { private FunctionDescProto findFunction(String signature, FunctionType type, List params, boolean strictTypeCheck) { if (functions.containsKey(signature)) { - if (strictTypeCheck) { for (FunctionDescProto existing : functions.get(signature)) { if (existing.getType() == type && existing.getParameterTypesList().equals(params)) { @@ -887,7 +884,8 @@ public class CatalogServer extends AbstractService { } } else { for (FunctionDescProto existing : functions.get(signature)) { - if (existing.getParameterTypesList() != null && compareDataType(params, existing.getParameterTypesList())) { + if (existing.getParameterTypesList() != null && + CatalogUtil.isMatchedFunction(existing.getParameterTypesList(), params)) { return existing; } } @@ -900,91 +898,6 @@ public class CatalogServer extends AbstractService { return findFunction(target.getSignature(), target.getType(), target.getParameterTypesList(), strictTypeCheck); } - private boolean compareDataType(final List params, - final List existing) { - if (params.size() != existing.size()) { - return false; - } else { - - for (int index = 0; index < existing.size(); index++) { - Type existingDataType = existing.get(index).getType(); - Type paramDataType = params.get(index).getType(); - if (!(existingDataType.equals(paramDataType) || isCompatibleType(existingDataType, paramDataType))) { - return false; - } - } - } - return true; - } - - private boolean isCompatibleType(final Type exitingType, final Type argType) { - boolean flag = false; - if (argType == Type.NULL_TYPE) { - flag = true; - } else if (exitingType == Type.ANY) { - flag = true; - } else if (argType.getNumber() > exitingType.getNumber()) { - // NO POINT IN GOING FORWARD BECAUSE THE DATA TYPE CANNOT BE UPPER CASTED - flag = false; - } else { - //argType.getNumber() < exitingType.getNumber() - int exitingTypeNumber = exitingType.getNumber(); - int argTypeNumber = argType.getNumber(); - - if (Type.INT1.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.INT8.getNumber()) { - // INT1 ==> INT2 ==> INT4 ==> INT8 - if (Type.INT1.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.UINT1.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.UINT8.getNumber()) { - // UINT1 ==> UINT2 ==> UINT4 ==> UINT8 - if (Type.UINT1.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.FLOAT4.getNumber() <= exitingTypeNumber && exitingTypeNumber <= Type.NUMERIC.getNumber()) { - //FLOAT4 ==> FLOAT8 ==> NUMERIC ==> DECIMAL - if (Type.FLOAT4.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.CHAR.getNumber() <= exitingTypeNumber && Type.TEXT.getNumber() <= exitingTypeNumber) { - //CHAR ==> NCHAR ==> VARCHAR ==> NVARCHAR ==> TEXT - if (Type.CHAR.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.BIT.getNumber() <= exitingTypeNumber && Type.VARBINARY.getNumber() <= exitingTypeNumber) { - // BIT ==> VARBIT ==> BINARY ==> VARBINARY - if (argTypeNumber >= 41 && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.INT1_ARRAY.getNumber() <= exitingTypeNumber - && Type.INT8_ARRAY.getNumber() <= exitingTypeNumber) { - // INT1_ARRAY ==> INT2_ARRAY ==> INT4_ARRAY ==> INT8_ARRAY - if (Type.INT1_ARRAY.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.UINT1_ARRAY.getNumber() <= exitingTypeNumber - && Type.UINT8_ARRAY.getNumber() <= exitingTypeNumber) { - // UINT1_ARRAY ==> UINT2_ARRAY ==> UINT4_ARRAY ==> UINT8_ARRAY - if (Type.UINT1_ARRAY.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.FLOAT4_ARRAY.getNumber() <= exitingTypeNumber - && Type.FLOAT8_ARRAY.getNumber() <= exitingTypeNumber) { - // FLOAT4_ARRAY ==> FLOAT8_ARRAY ==> NUMERIC_ARRAY ==> DECIMAL_ARRAY - if (Type.FLOAT4.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } else if (Type.CHAR_ARRAY.getNumber() <= exitingTypeNumber - && Type.TEXT_ARRAY.getNumber() <= exitingTypeNumber) { - // CHAR_ARRAY ==> FLOAT8_ARRAY ==> NUMERIC_ARRAY ==> DECIMAL_ARRAY - if (Type.TEXT_ARRAY.getNumber() <= argTypeNumber && argTypeNumber <= exitingTypeNumber) { - flag = true; - } - } - } - return flag; - } - @Override public BoolProto createFunction(RpcController controller, FunctionDescProto funcDesc) throws ServiceException { http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index e60cc09..32ea83b 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -886,8 +886,8 @@ public class TestCatalog { CatalogUtil.newSimpleDataTypeArray(Type.FLOAT8, Type.INT4)); assertTrue(catalog.createFunction(meta)); - //UPGRADE TO DECIMAL WILL FAIL ==> LOOK AT FIRST PARAM BELOW - catalog.getFunction("testfloatinvalid", CatalogUtil.newSimpleDataTypeArray(Type.DECIMAL, Type.INT4)); + // UPGRADE TO DECIMAL WILL FAIL ==> LOOK AT FIRST PARAM BELOW + catalog.getFunction("testfloatinvalid", CatalogUtil.newSimpleDataTypeArray(Type.NUMERIC, Type.INT4)); } @Test http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java b/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java index 1573978..056eb2c 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java @@ -67,7 +67,7 @@ public class ResultSetUtil { return "float"; case FLOAT8: return "float8"; - case DECIMAL: + case NUMERIC: return "numeric"; case VARBINARY: return "bytea"; @@ -100,8 +100,8 @@ public class ResultSetUtil { return Types.FLOAT; case FLOAT8: return Types.DOUBLE; - case DECIMAL: - return Types.DECIMAL; + case NUMERIC: + return Types.NUMERIC; case DATE: return Types.TIMESTAMP; case VARCHAR: http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java index 577fe0e..b3c53e1 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java @@ -177,7 +177,6 @@ public abstract class TajoResultSetBase implements ResultSet { case VARCHAR: return d.asChars(); case FLOAT4: return d.asFloat4(); case FLOAT8: return d.asFloat8(); - case DECIMAL: case NUMERIC: return d.asFloat8(); default: return d.asChars(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-common/src/main/proto/DataTypes.proto ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/proto/DataTypes.proto b/tajo-common/src/main/proto/DataTypes.proto index 9b5b98f..fb9c957 100644 --- a/tajo-common/src/main/proto/DataTypes.proto +++ b/tajo-common/src/main/proto/DataTypes.proto @@ -39,7 +39,6 @@ enum Type { FLOAT8 = 11; // variable-precision, inexact [8 bytes] NUMERIC = 12; // variable length - DECIMAL = 13; // variable length CHAR = 21; // fixed-width n-character string NCHAR = 22; // fixed width string supporting an international character set @@ -82,7 +81,6 @@ enum Type { FLOAT8_ARRAY = 111; NUMERIC_ARRAY = 112; - DECIMAL_ARRAY = 113; CHAR_ARRAY = 121; NCHAR_ARRAY = 122; http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java new file mode 100644 index 0000000..98abb02 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.builtin; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.storage.Tuple; + +public abstract class Coalesce extends GeneralFunction { + public Coalesce(Column[] definedArgs) { + super(definedArgs); + } + + @Override + public Datum eval(Tuple params) { + int paramSize = params.size(); + for (int i = 0; i < paramSize; i++) { + Datum datum = params.get(i); + if (datum instanceof NullDatum) { + continue; + } + return datum; + } + return NullDatum.get(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceDouble.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceDouble.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceDouble.java new file mode 100644 index 0000000..3e94150 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceDouble.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.builtin; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; + +@Description( + functionName = "coalesce", + description = "Returns the first of its arguments that is not null.", + detail = "Like a CASE expression, COALESCE only evaluates the arguments that are needed to determine the result; " + + "that is, arguments to the right of the first non-null argument are not evaluated", + example = "> SELECT coalesce(null, null, 10.0);\n" + + "10.0", + returnType = TajoDataTypes.Type.FLOAT8, + paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.FLOAT8, TajoDataTypes.Type.FLOAT8_ARRAY})} +) +public class CoalesceDouble extends Coalesce { + public CoalesceDouble() { + super(new Column[] { + new Column("column", TajoDataTypes.Type.FLOAT8), + new Column("params", TajoDataTypes.Type.FLOAT8_ARRAY), + }); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceLong.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceLong.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceLong.java new file mode 100644 index 0000000..5d55255 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceLong.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.builtin; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; + +@Description( + functionName = "coalesce", + description = "Returns the first of its arguments that is not null.", + detail = "Like a CASE expression, COALESCE only evaluates the arguments that are needed to determine the result; " + + "that is, arguments to the right of the first non-null argument are not evaluated", + example = "> SELECT coalesce(null, null, 10);\n" + + "10", + returnType = TajoDataTypes.Type.INT8, + paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.INT8, TajoDataTypes.Type.INT8_ARRAY})} +) +public class CoalesceLong extends Coalesce { + + public CoalesceLong() { + super(new Column[] { + new Column("column", TajoDataTypes.Type.INT8), + new Column("params", TajoDataTypes.Type.INT8_ARRAY), + }); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceString.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceString.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceString.java new file mode 100644 index 0000000..50e4786 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/builtin/CoalesceString.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.builtin; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; + +@Description( + functionName = "coalesce", + description = "Returns the first of its arguments that is not null.", + detail = "Like a CASE expression, COALESCE only evaluates the arguments that are needed to determine the result; " + + "that is, arguments to the right of the first non-null argument are not evaluated", + example = "> SELECT coalesce(null, null, 'default');\n" + + "default", + returnType = TajoDataTypes.Type.TEXT, + paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TEXT_ARRAY})} +) +public class CoalesceString extends Coalesce { + + public CoalesceString() { + super(new Column[] { + new Column("column", TajoDataTypes.Type.TEXT), + new Column("params", TajoDataTypes.Type.TEXT_ARRAY), + }); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java index c734583..60e9685 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/HiveQLAnalyzer.java @@ -1495,7 +1495,7 @@ public class HiveQLAnalyzer extends HiveQLParserBaseVisitor { } else if (primitiveType.KW_DOUBLE() != null) { typeDefinition = new DataTypeExpr(TajoDataTypes.Type.FLOAT8.name()); } else if (primitiveType.KW_DECIMAL() != null) { - typeDefinition = new DataTypeExpr(TajoDataTypes.Type.DECIMAL.name()); + typeDefinition = new DataTypeExpr(TajoDataTypes.Type.NUMERIC.name()); } else if (primitiveType.KW_BOOLEAN() != null) { typeDefinition = new DataTypeExpr(TajoDataTypes.Type.BOOLEAN.name()); } else if (primitiveType.KW_DATE() != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 5d75edd..cb356f8 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1193,11 +1193,10 @@ public class SQLAnalyzer extends SQLParserBaseVisitor { } else if (exactType.NUMERIC() != null) { typeDefinition = new DataTypeExpr(Type.NUMERIC.name()); } else if (exactType.DECIMAL() != null || exactType.DEC() != null) { - typeDefinition = new DataTypeExpr(Type.DECIMAL.name()); + typeDefinition = new DataTypeExpr(Type.NUMERIC.name()); } - if (typeDefinition.getTypeName().equals(Type.NUMERIC.name()) || - typeDefinition.getTypeName().equals(Type.DECIMAL.name())) { + if (typeDefinition.getTypeName().equals(Type.NUMERIC.name())) { if (exactType.precision_param() != null) { if (exactType.precision_param().scale != null) { typeDefinition.setScale( http://git-wip-us.apache.org/repos/asf/tajo/blob/014f9f3e/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java index 46f5672..551393c 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java @@ -152,7 +152,7 @@ public class ExprsVerifier extends BasicEvalNodeVisitor