From commits-return-20800-archive-asf-public=cust-asf.ponee.io@phoenix.apache.org Thu Apr 26 23:43:55 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 475941807A1 for ; Thu, 26 Apr 2018 23:43:54 +0200 (CEST) Received: (qmail 97000 invoked by uid 500); 26 Apr 2018 21:43:53 -0000 Mailing-List: contact commits-help@phoenix.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@phoenix.apache.org Delivered-To: mailing list commits@phoenix.apache.org Received: (qmail 96832 invoked by uid 99); 26 Apr 2018 21:43:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 26 Apr 2018 21:43:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 23630F5E1C; Thu, 26 Apr 2018 21:43:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: pboado@apache.org To: commits@phoenix.apache.org Date: Thu, 26 Apr 2018 21:43:59 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [7/9] phoenix git commit: PHOENIX-4645 PhoenixStorageHandler doesn't handle correctly data/timestamp in push down predicate when engine is tez.(Sergey Soldatov) PHOENIX-4645 PhoenixStorageHandler doesn't handle correctly data/timestamp in push down predicate when engine is tez.(Sergey Soldatov) Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1dc730fb Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1dc730fb Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1dc730fb Branch: refs/heads/4.x-cdh5.13 Commit: 1dc730fb141ca22f00c760400473b95c7effc6f0 Parents: 33ed01d Author: Rajeshbabu Chintaguntla Authored: Tue Apr 24 06:55:57 2018 +0100 Committer: Pedro Boado Committed: Thu Apr 26 22:28:57 2018 +0100 ---------------------------------------------------------------------- .../apache/phoenix/hive/HivePhoenixStoreIT.java | 27 ++++++++++++++++++++ .../PhoenixStorageHandlerConstants.java | 8 +++--- .../phoenix/hive/query/PhoenixQueryBuilder.java | 4 +-- 3 files changed, 33 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/1dc730fb/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java index edf72f1..5828ed0 100644 --- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java @@ -311,6 +311,33 @@ public class HivePhoenixStoreIT extends BaseHivePhoenixStoreIT { assert (rs.getString(2).equalsIgnoreCase("part1")); assert (rs.getDouble(3) == 200); } + } + + @Test + public void testTimestampPredicate() throws Exception { + String testName = "testTimeStampPredicate"; + hbaseTestUtil.getTestFileSystem().createNewFile(new Path(hiveLogDir, testName + ".out")); + createFile("10\t2013-01-02 01:01:01.123456\n", new Path(hiveOutputDir, testName + ".out").toString()); + createFile(StringUtil.EMPTY_STRING, new Path(hiveLogDir, testName + ".out").toString()); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE timeStampTable(ID int,ts TIMESTAMP)" + HiveTestUtil.CRLF + + " STORED BY \"org.apache.phoenix.hive.PhoenixStorageHandler\"" + HiveTestUtil + .CRLF + + " TBLPROPERTIES(" + HiveTestUtil.CRLF + + " 'phoenix.hbase.table.name'='TIMESTAMPTABLE'," + HiveTestUtil.CRLF + + " 'phoenix.zookeeper.znode.parent'='/hbase'," + HiveTestUtil.CRLF + + " 'phoenix.zookeeper.quorum'='localhost'," + HiveTestUtil.CRLF + + " 'phoenix.zookeeper.client.port'='" + + hbaseTestUtil.getZkCluster().getClientPort() + "'," + HiveTestUtil.CRLF + + " 'phoenix.column.mapping' = 'id:ID, ts:TS'," + HiveTestUtil.CRLF + + " 'phoenix.rowkeys'='id');" + HiveTestUtil.CRLF); + sb.append("INSERT INTO TABLE timeStampTable VALUES (10, \"2013-01-02 01:01:01.123456\");" + HiveTestUtil.CRLF); + sb.append("SELECT * from timeStampTable WHERE ts between '2013-01-02 01:01:01.123455' and " + + " '2013-01-02 12:01:02.123457789' AND id = 10;" + HiveTestUtil.CRLF); + + String fullPath = new Path(hbaseTestUtil.getDataTestDir(), testName).toString(); + createFile(sb.toString(), fullPath); + runTest(testName, fullPath); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/1dc730fb/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java index 07c374e..e3c7d84 100644 --- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/constants/PhoenixStorageHandlerConstants.java @@ -90,13 +90,13 @@ public class PhoenixStorageHandlerConstants { public static final String PATERN_MARKER = "$targetPattern$"; public static final String DATE_PATTERN = "'?\\d{4}-\\d{2}-\\d{2}'?"; public static final String TIMESTAMP_PATTERN = "'?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\" + - ".?\\d{0,3}'?"; - public static final String COMMON_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*" + + ".?\\d{0,9}'?"; + public static final String COMMON_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*" + "(=|>|<|<=|>=)\\s*(" + PATERN_MARKER + "))"; - public static final String BETWEEN_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*(" + + public static final String BETWEEN_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*(" + "(?i)not)?\\s*(?i)between\\s*(" + PATERN_MARKER + ")\\s*(?i)and\\s*(" + PATERN_MARKER + "))"; - public static final String IN_OPERATOR_PATTERN = "(\\(?" + COLUMNE_MARKER + "\\)?\\s*((?i)" + + public static final String IN_OPERATOR_PATTERN = "(\\(?\"?" + COLUMNE_MARKER + "\"?\\)?\\s*((?i)" + "not)?\\s*(?i)in\\s*\\((" + PATERN_MARKER + ",?\\s*)+\\))"; public static final String FUNCTION_VALUE_MARKER = "$value$"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/1dc730fb/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java index eb336dc..ab409ad 100644 --- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java +++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java @@ -255,10 +255,10 @@ public class PhoenixQueryBuilder { if (PhoenixStorageHandlerConstants.DATE_TYPE.equals( columnTypeMap.get(columnName).getTypeName())) { - whereClause = applyDateFunctionUsingRegex(whereClause, columnName); + whereClause = applyDateFunctionUsingRegex(whereClause, column); } else if (PhoenixStorageHandlerConstants.TIMESTAMP_TYPE.equals( columnTypeMap.get(columnName).getTypeName())) { - whereClause = applyTimestampFunctionUsingRegex(whereClause, columnName); + whereClause = applyTimestampFunctionUsingRegex(whereClause, column); } } }