Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 76A131893A for ; Tue, 28 Jul 2015 19:49:05 +0000 (UTC) Received: (qmail 71489 invoked by uid 500); 28 Jul 2015 19:49:05 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 71107 invoked by uid 500); 28 Jul 2015 19:49:05 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 70597 invoked by uid 99); 28 Jul 2015 19:49:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 28 Jul 2015 19:49:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 62FEBE35DB; Tue, 28 Jul 2015 19:49:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: omalley@apache.org To: commits@hive.apache.org Date: Tue, 28 Jul 2015 19:49:05 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/5] hive git commit: HIVE-10799. Refactor the SearchArgumentFactory to remove the AST-specific factory. (omalley reviewed by prasanth_j) http://git-wip-us.apache.org/repos/asf/hive/blob/c178a6e9/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java index 46ce49c..3a51f4a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java @@ -19,27 +19,23 @@ package org.apache.hadoop.hive.ql.io.sarg; import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertNull; import static junit.framework.Assert.assertTrue; import com.google.common.collect.Sets; import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl.ExpressionBuilder; import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl.PredicateLeafImpl; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.junit.Test; import java.beans.XMLDecoder; import java.io.ByteArrayInputStream; import java.io.UnsupportedEncodingException; import java.lang.reflect.Field; -import java.math.BigDecimal; +import java.sql.Date; import java.sql.Timestamp; import java.util.List; import java.util.Set; @@ -92,96 +88,103 @@ public class TestSearchArgumentImpl { @Test public void testNotPushdown() throws Exception { - assertEquals("leaf-1", ExpressionBuilder.pushDownNot(leaf(1)).toString()); + assertEquals("leaf-1", SearchArgumentImpl.BuilderImpl.pushDownNot(leaf(1)) + .toString()); assertEquals("(not leaf-1)", - ExpressionBuilder.pushDownNot(not(leaf(1))).toString()); + SearchArgumentImpl.BuilderImpl.pushDownNot(not(leaf(1))).toString()); assertEquals("leaf-1", - ExpressionBuilder.pushDownNot(not(not(leaf(1)))).toString()); + SearchArgumentImpl.BuilderImpl.pushDownNot(not(not(leaf(1)))) + .toString()); assertEquals("(not leaf-1)", - ExpressionBuilder.pushDownNot(not(not(not(leaf(1))))).toString()); + SearchArgumentImpl.BuilderImpl.pushDownNot(not(not(not(leaf(1))))). + toString()); assertEquals("(or leaf-1 (not leaf-2))", - ExpressionBuilder.pushDownNot(not(and(not(leaf(1)), + SearchArgumentImpl.BuilderImpl.pushDownNot(not(and(not(leaf(1)), leaf(2)))).toString()); assertEquals("(and (not leaf-1) leaf-2)", - ExpressionBuilder.pushDownNot(not(or(leaf(1), + SearchArgumentImpl.BuilderImpl.pushDownNot(not(or(leaf(1), not(leaf(2))))).toString()); assertEquals("(or (or (not leaf-1) leaf-2) leaf-3)", - ExpressionBuilder.pushDownNot(or(not(and(leaf(1), not(leaf(2)))), + SearchArgumentImpl.BuilderImpl.pushDownNot(or(not(and(leaf(1), + not(leaf(2)))), not(not(leaf(3))))).toString()); - assertEquals("NO", ExpressionBuilder.pushDownNot( + assertEquals("NO", SearchArgumentImpl.BuilderImpl.pushDownNot( not(constant(TruthValue.YES))).toString()); - assertEquals("YES", ExpressionBuilder.pushDownNot( + assertEquals("YES", SearchArgumentImpl.BuilderImpl.pushDownNot( not(constant(TruthValue.NO))).toString()); - assertEquals("NULL", ExpressionBuilder.pushDownNot( + assertEquals("NULL", SearchArgumentImpl.BuilderImpl.pushDownNot( not(constant(TruthValue.NULL))).toString()); - assertEquals("YES_NO", ExpressionBuilder.pushDownNot( + assertEquals("YES_NO", SearchArgumentImpl.BuilderImpl.pushDownNot( not(constant(TruthValue.YES_NO))).toString()); - assertEquals("YES_NULL", ExpressionBuilder.pushDownNot( + assertEquals("YES_NULL", SearchArgumentImpl.BuilderImpl.pushDownNot( not(constant(TruthValue.NO_NULL))).toString()); - assertEquals("NO_NULL", ExpressionBuilder.pushDownNot( + assertEquals("NO_NULL", SearchArgumentImpl.BuilderImpl.pushDownNot( not(constant(TruthValue.YES_NULL))).toString()); - assertEquals("YES_NO_NULL", ExpressionBuilder.pushDownNot( + assertEquals("YES_NO_NULL", SearchArgumentImpl.BuilderImpl.pushDownNot( not(constant(TruthValue.YES_NO_NULL))).toString()); } @Test public void testFlatten() throws Exception { - assertEquals("leaf-1", ExpressionBuilder.flatten(leaf(1)).toString()); + assertEquals("leaf-1", SearchArgumentImpl.BuilderImpl.flatten(leaf(1)).toString()); assertEquals("NO", - ExpressionBuilder.flatten(constant(TruthValue.NO)).toString()); + SearchArgumentImpl.BuilderImpl.flatten(constant(TruthValue.NO)).toString()); assertEquals("(not (not leaf-1))", - ExpressionBuilder.flatten(not(not(leaf(1)))).toString()); + SearchArgumentImpl.BuilderImpl.flatten(not(not(leaf(1)))).toString()); assertEquals("(and leaf-1 leaf-2)", - ExpressionBuilder.flatten(and(leaf(1), leaf(2))).toString()); + SearchArgumentImpl.BuilderImpl.flatten(and(leaf(1), leaf(2))).toString()); assertEquals("(and (or leaf-1 leaf-2) leaf-3)", - ExpressionBuilder.flatten(and(or(leaf(1), leaf(2)), leaf(3)) + SearchArgumentImpl.BuilderImpl.flatten(and(or(leaf(1), leaf(2)), leaf(3)) ).toString()); assertEquals("(and leaf-1 leaf-2 leaf-3 leaf-4)", - ExpressionBuilder.flatten(and(and(leaf(1), leaf(2)), + SearchArgumentImpl.BuilderImpl.flatten(and(and(leaf(1), leaf(2)), and(leaf(3), leaf(4)))).toString()); assertEquals("(or leaf-1 leaf-2 leaf-3 leaf-4)", - ExpressionBuilder.flatten(or(leaf(1), or(leaf(2), or(leaf(3), + SearchArgumentImpl.BuilderImpl.flatten(or(leaf(1), or(leaf(2), or(leaf(3), leaf(4))))).toString()); assertEquals("(or leaf-1 leaf-2 leaf-3 leaf-4)", - ExpressionBuilder.flatten(or(or(or(leaf(1), leaf(2)), leaf(3)), + SearchArgumentImpl.BuilderImpl.flatten(or(or(or(leaf(1), leaf(2)), leaf(3)), leaf(4))).toString()); assertEquals("(or leaf-1 leaf-2 leaf-3 leaf-4 leaf-5 leaf-6)", - ExpressionBuilder.flatten(or(or(leaf(1), or(leaf(2), leaf(3))), + SearchArgumentImpl.BuilderImpl.flatten(or(or(leaf(1), or(leaf(2), leaf(3))), or(or(leaf(4), leaf(5)), leaf(6)))).toString()); assertEquals("(and (not leaf-1) leaf-2 (not leaf-3) leaf-4 (not leaf-5) leaf-6)", - ExpressionBuilder.flatten(and(and(not(leaf(1)), and(leaf(2), - not(leaf(3)))), and(and(leaf(4), not(leaf(5))), leaf(6))) + SearchArgumentImpl.BuilderImpl.flatten(and(and(not(leaf(1)), and(leaf(2), + not(leaf(3)))), and(and(leaf(4), not(leaf(5))), leaf(6))) ).toString()); assertEquals("(not (and leaf-1 leaf-2 leaf-3))", - ExpressionBuilder.flatten(not(and(leaf(1), and(leaf(2), leaf(3)))) + SearchArgumentImpl.BuilderImpl.flatten(not(and(leaf(1), and(leaf(2), leaf(3)))) ).toString()); } @Test public void testFoldMaybe() throws Exception { assertEquals("(and leaf-1)", - ExpressionBuilder.foldMaybe(and(leaf(1), + SearchArgumentImpl.BuilderImpl.foldMaybe(and(leaf(1), constant(TruthValue.YES_NO_NULL))).toString()); assertEquals("(and leaf-1 leaf-2)", - ExpressionBuilder.foldMaybe(and(leaf(1), + SearchArgumentImpl.BuilderImpl.foldMaybe(and(leaf(1), constant(TruthValue.YES_NO_NULL), leaf(2))).toString()); assertEquals("(and leaf-1 leaf-2)", - ExpressionBuilder.foldMaybe(and(constant(TruthValue.YES_NO_NULL), - leaf(1), leaf(2), constant(TruthValue.YES_NO_NULL))).toString()); + SearchArgumentImpl.BuilderImpl. + foldMaybe(and(constant(TruthValue.YES_NO_NULL), + leaf(1), leaf(2), constant(TruthValue.YES_NO_NULL))).toString()); assertEquals("YES_NO_NULL", - ExpressionBuilder.foldMaybe(and(constant(TruthValue.YES_NO_NULL), - constant(TruthValue.YES_NO_NULL))).toString()); + SearchArgumentImpl.BuilderImpl. + foldMaybe(and(constant(TruthValue.YES_NO_NULL), + constant(TruthValue.YES_NO_NULL))).toString()); assertEquals("YES_NO_NULL", - ExpressionBuilder.foldMaybe(or(leaf(1), - constant(TruthValue.YES_NO_NULL))).toString()); + SearchArgumentImpl.BuilderImpl. + foldMaybe(or(leaf(1), + constant(TruthValue.YES_NO_NULL))).toString()); assertEquals("(or leaf-1 (and leaf-2))", - ExpressionBuilder.foldMaybe(or(leaf(1), + SearchArgumentImpl.BuilderImpl.foldMaybe(or(leaf(1), and(leaf(2), constant(TruthValue.YES_NO_NULL)))).toString()); assertEquals("(and leaf-1)", - ExpressionBuilder.foldMaybe(and(or(leaf(2), + SearchArgumentImpl.BuilderImpl.foldMaybe(and(or(leaf(2), constant(TruthValue.YES_NO_NULL)), leaf(1))).toString()); - assertEquals("(and leaf-100)", ExpressionBuilder.foldMaybe( - ExpressionBuilder.convertToCNF(and(leaf(100), + assertEquals("(and leaf-100)", SearchArgumentImpl.BuilderImpl.foldMaybe( + SearchArgumentImpl.BuilderImpl.convertToCNF(and(leaf(100), or(and(leaf(0), leaf(1)), and(leaf(2), leaf(3)), and(leaf(4), leaf(5)), @@ -195,28 +198,31 @@ public class TestSearchArgumentImpl { @Test public void testCNF() throws Exception { - assertEquals("leaf-1", ExpressionBuilder.convertToCNF(leaf(1)).toString()); - assertEquals("NO", ExpressionBuilder.convertToCNF( + assertEquals("leaf-1", SearchArgumentImpl.BuilderImpl.convertToCNF(leaf(1)). + toString()); + assertEquals("NO", SearchArgumentImpl.BuilderImpl.convertToCNF( constant(TruthValue.NO)).toString()); - assertEquals("(not leaf-1)", ExpressionBuilder.convertToCNF( + assertEquals("(not leaf-1)", SearchArgumentImpl.BuilderImpl.convertToCNF( not(leaf(1))).toString()); - assertEquals("(and leaf-1 leaf-2)", ExpressionBuilder.convertToCNF( - and(leaf(1), leaf(2))).toString()); - assertEquals("(or (not leaf-1) leaf-2)", ExpressionBuilder.convertToCNF( - or(not(leaf(1)), leaf(2))).toString()); + assertEquals("(and leaf-1 leaf-2)", SearchArgumentImpl.BuilderImpl. + convertToCNF( + and(leaf(1), leaf(2))).toString()); + assertEquals("(or (not leaf-1) leaf-2)", SearchArgumentImpl.BuilderImpl. + convertToCNF( + or(not(leaf(1)), leaf(2))).toString()); assertEquals("(and (or leaf-1 leaf-2) (not leaf-3))", - ExpressionBuilder.convertToCNF( + SearchArgumentImpl.BuilderImpl.convertToCNF( and(or(leaf(1), leaf(2)), not(leaf(3)))).toString()); assertEquals("(and (or leaf-1 leaf-3) (or leaf-2 leaf-3)" + " (or leaf-1 leaf-4) (or leaf-2 leaf-4))", - ExpressionBuilder.convertToCNF( + SearchArgumentImpl.BuilderImpl.convertToCNF( or(and(leaf(1), leaf(2)), and(leaf(3), leaf(4)))).toString()); assertEquals("(and" + " (or leaf-1 leaf-5) (or leaf-2 leaf-5)" + " (or leaf-3 leaf-5) (or leaf-4 leaf-5)" + " (or leaf-1 leaf-6) (or leaf-2 leaf-6)" + " (or leaf-3 leaf-6) (or leaf-4 leaf-6))", - ExpressionBuilder.convertToCNF( + SearchArgumentImpl.BuilderImpl.convertToCNF( or(and(leaf(1), leaf(2), leaf(3), leaf(4)), and(leaf(5), leaf(6)))).toString()); assertEquals("(and" + @@ -224,7 +230,7 @@ public class TestSearchArgumentImpl { " (or leaf-5 leaf-6 (not leaf-7) leaf-2 leaf-3)" + " (or leaf-5 leaf-6 (not leaf-7) leaf-1 leaf-4)" + " (or leaf-5 leaf-6 (not leaf-7) leaf-2 leaf-4))", - ExpressionBuilder.convertToCNF( + SearchArgumentImpl.BuilderImpl.convertToCNF( or(and(leaf(1), leaf(2)), and(leaf(3), leaf(4)), or(leaf(5), leaf(6)), @@ -248,33 +254,37 @@ public class TestSearchArgumentImpl { " (or leaf-8 leaf-0 leaf-5 leaf-7)" + " (or leaf-8 leaf-1 leaf-5 leaf-7)" + " (or leaf-8 leaf-2 leaf-5 leaf-7))", - ExpressionBuilder.convertToCNF(or(and(leaf(0), leaf(1), leaf(2)), + SearchArgumentImpl.BuilderImpl.convertToCNF(or(and(leaf(0), leaf(1), + leaf(2)), and(leaf(3), leaf(4), leaf(5)), and(leaf(6), leaf(7)), leaf(8))).toString()); - assertEquals("YES_NO_NULL", ExpressionBuilder.convertToCNF(or(and(leaf(0), leaf(1)), - and(leaf(2), leaf(3)), - and(leaf(4), leaf(5)), - and(leaf(6), leaf(7)), - and(leaf(8), leaf(9)), - and(leaf(10), leaf(11)), - and(leaf(12), leaf(13)), - and(leaf(14), leaf(15)), - and(leaf(16), leaf(17)))).toString()); - assertEquals("(and leaf-100 YES_NO_NULL)", ExpressionBuilder.convertToCNF(and(leaf(100), - or(and(leaf(0), leaf(1)), - and(leaf(2), leaf(3)), - and(leaf(4), leaf(5)), - and(leaf(6), leaf(7)), - and(leaf(8), leaf(9)), - and(leaf(10), leaf(11)), - and(leaf(12), leaf(13)), - and(leaf(14), leaf(15)), - and(leaf(16), leaf(17))))).toString()); - assertNoSharedNodes(ExpressionBuilder.convertToCNF(or(and(leaf(0), leaf(1), leaf(2)), - and(leaf(3), leaf(4), leaf(5)), - and(leaf(6), leaf(7)), - leaf(8))), Sets.newIdentityHashSet()); + assertEquals("YES_NO_NULL", SearchArgumentImpl.BuilderImpl. + convertToCNF(or(and(leaf(0), leaf(1)), + and(leaf(2), leaf(3)), + and(leaf(4), leaf(5)), + and(leaf(6), leaf(7)), + and(leaf(8), leaf(9)), + and(leaf(10), leaf(11)), + and(leaf(12), leaf(13)), + and(leaf(14), leaf(15)), + and(leaf(16), leaf(17)))).toString()); + assertEquals("(and leaf-100 YES_NO_NULL)", SearchArgumentImpl.BuilderImpl. + convertToCNF(and(leaf(100), + or(and(leaf(0), leaf(1)), + and(leaf(2), leaf(3)), + and(leaf(4), leaf(5)), + and(leaf(6), leaf(7)), + and(leaf(8), leaf(9)), + and(leaf(10), leaf(11)), + and(leaf(12), leaf(13)), + and(leaf(14), leaf(15)), + and(leaf(16), leaf(17))))).toString()); + assertNoSharedNodes(SearchArgumentImpl.BuilderImpl. + convertToCNF(or(and(leaf(0), leaf(1), leaf(2)), + and(leaf(3), leaf(4), leaf(5)), + and(leaf(6), leaf(7)), + leaf(8))), Sets.newIdentityHashSet()); } private static void assertNoSharedNodes(ExpressionTree tree, @@ -310,2596 +320,6 @@ public class TestSearchArgumentImpl { } } - @Test - public void testExpression1() throws Exception { - // first_name = 'john' or - // 'greg' < first_name or - // 'alan' > first_name or - // id > 12 or - // 13 < id or - // id < 15 or - // 16 > id or - // (id <=> 30 and first_name <=> 'owen') - String exprStr = " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " string \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " john \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " boolean \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " greg \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " alan \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " int \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 12 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 13 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 15 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 16 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 30 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " owen \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n"; - - SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgumentFactory.create(getFuncDesc(exprStr)); - List leaves = sarg.getLeaves(); - assertEquals(9, leaves.size()); - - FilterPredicate p = ParquetRecordReaderWrapper.toFilterPredicate(sarg); - String[] conditions = new String[]{ - "eq(first_name, Binary{\"john\"})", /* first_name = 'john' */ - "not(lteq(first_name, Binary{\"greg\"}))", /* 'greg' < first_name */ - "lt(first_name, Binary{\"alan\"})", /* 'alan' > first_name */ - "not(lteq(id, 12))", /* id > 12 or */ - "not(lteq(id, 13))", /* 13 < id or */ - "lt(id, 15)", /* id < 15 or */ - "lt(id, 16)", /* 16 > id or */ - "eq(id, 30)", /* id <=> 30 */ - "eq(first_name, Binary{\"owen\"})" /* first_name <=> 'owen' */ - }; - String expected = String - .format("and(or(or(or(or(or(or(or(%1$s, %2$s), %3$s), %4$s), %5$s), %6$s), %7$s), %8$s), " + - "or(or(or(or(or(or(or(%1$s, %2$s), %3$s), %4$s), %5$s), %6$s), %7$s), %9$s))", conditions); - assertEquals(expected, p.toString()); - - PredicateLeaf leaf = leaves.get(0); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator()); - assertEquals("first_name", leaf.getColumnName()); - assertEquals("john", leaf.getLiteral()); - - leaf = leaves.get(1); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator()); - assertEquals("first_name", leaf.getColumnName()); - assertEquals("greg", leaf.getLiteral()); - - leaf = leaves.get(2); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); - assertEquals("first_name", leaf.getColumnName()); - assertEquals("alan", leaf.getLiteral()); - - leaf = leaves.get(3); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(12, leaf.getLiteral()); - - leaf = leaves.get(4); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(13, leaf.getLiteral()); - - leaf = leaves.get(5); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(15, leaf.getLiteral()); - - leaf = leaves.get(6); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(16, leaf.getLiteral()); - - leaf = leaves.get(7); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.NULL_SAFE_EQUALS, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(30, leaf.getLiteral()); - - leaf = leaves.get(8); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.NULL_SAFE_EQUALS, leaf.getOperator()); - assertEquals("first_name", leaf.getColumnName()); - assertEquals("owen", leaf.getLiteral()); - - assertEquals("(and (or leaf-0 (not leaf-1) leaf-2 (not leaf-3)" + - " (not leaf-4) leaf-5 leaf-6 leaf-7)" + - " (or leaf-0 (not leaf-1) leaf-2 (not leaf-3)" + - " (not leaf-4) leaf-5 leaf-6 leaf-8))", - sarg.getExpression().toString()); - assertNoSharedNodes(sarg.getExpression(), - Sets.newIdentityHashSet()); - } - - @Test - public void testExpression2() throws Exception { - /* first_name is null or - first_name <> 'sue' or - id >= 12 or - id <= 4; */ - String exprStr = " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " string \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " boolean \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " sue \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " int \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 12 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 4 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n"; - - SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgumentFactory.create(getFuncDesc(exprStr)); - List leaves = sarg.getLeaves(); - assertEquals(4, leaves.size()); - - String[] conditions = new String[]{ - "eq(first_name, null)", /* first_name is null */ - "not(eq(first_name, Binary{\"sue\"}))", /* first_name <> 'sue' */ - "not(lt(id, 12))", /* id >= 12 */ - "lteq(id, 4)" /* id <= 4 */ - }; - - FilterPredicate p = ParquetRecordReaderWrapper.toFilterPredicate(sarg); - String expected = String.format("or(or(or(%1$s, %2$s), %3$s), %4$s)", conditions); - assertEquals(expected, p.toString()); - - PredicateLeaf leaf = leaves.get(0); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.IS_NULL, leaf.getOperator()); - assertEquals("first_name", leaf.getColumnName()); - assertEquals(null, leaf.getLiteral()); - assertEquals(null, leaf.getLiteralList()); - - leaf = leaves.get(1); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator()); - assertEquals("first_name", leaf.getColumnName()); - assertEquals("sue", leaf.getLiteral()); - - leaf = leaves.get(2); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(12, leaf.getLiteral()); - - leaf = leaves.get(3); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.LESS_THAN_EQUALS, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(4, leaf.getLiteral()); - - assertEquals("(or leaf-0 (not leaf-1) (not leaf-2) leaf-3)", - sarg.getExpression().toString()); - assertNoSharedNodes(sarg.getExpression(), - Sets.newIdentityHashSet()); - assertEquals(TruthValue.NO, - sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.YES, - TruthValue.NO))); - assertEquals(TruthValue.YES, - sarg.evaluate(values(TruthValue.YES, TruthValue.YES, TruthValue.YES, - TruthValue.NO))); - assertEquals(TruthValue.YES, - sarg.evaluate(values(TruthValue.NO, TruthValue.NO, TruthValue.YES, - TruthValue.NO))); - assertEquals(TruthValue.YES, - sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.NO, - TruthValue.NO))); - assertEquals(TruthValue.YES, - sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.YES, - TruthValue.YES))); - assertEquals(TruthValue.NULL, - sarg.evaluate(values(TruthValue.NULL, TruthValue.YES, TruthValue.YES, - TruthValue.NO))); - assertEquals(TruthValue.NULL, - sarg.evaluate(values(TruthValue.NO, TruthValue.NULL, TruthValue.YES, - TruthValue.NO))); - assertEquals(TruthValue.NULL, - sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.NULL, - TruthValue.NO))); - assertEquals(TruthValue.NULL, - sarg.evaluate(values(TruthValue.NO, TruthValue.YES, TruthValue.YES, - TruthValue.NULL))); - assertEquals(TruthValue.YES_NO, - sarg.evaluate(values(TruthValue.NO, TruthValue.YES_NO, TruthValue.YES, - TruthValue.YES_NO))); - assertEquals(TruthValue.NO_NULL, - sarg.evaluate(values(TruthValue.NO, TruthValue.YES_NULL, TruthValue.YES, - TruthValue.NO_NULL))); - assertEquals(TruthValue.YES_NULL, - sarg.evaluate(values(TruthValue.YES_NULL, TruthValue.YES_NO_NULL, - TruthValue.YES, TruthValue.NULL))); - assertEquals(TruthValue.YES_NO_NULL, - sarg.evaluate(values(TruthValue.NO_NULL, TruthValue.YES_NO_NULL, - TruthValue.YES, TruthValue.NO))); - } - - @Test - public void testExpression3() throws Exception { - /* (id between 23 and 45) and - first_name = 'alan' and - substr('xxxxx', 3) == first_name and - 'smith' = last_name and - substr(first_name, 3) == 'yyy' */ - String exprStr = " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " boolean \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " false \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " int \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 23 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 45 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " string \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " alan \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " xxxxx \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 3 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + - " \n" + - " \n" + - " substr \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " smith \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " last_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 3 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " org.apache.hadoop.hive.ql.udf.UDFSubstr \n" + - " \n" + - " \n" + - " substr \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " yyy \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n"; - - SearchArgumentImpl sarg = - (SearchArgumentImpl) SearchArgumentFactory.create(getFuncDesc(exprStr)); - List leaves = sarg.getLeaves(); - assertEquals(3, leaves.size()); - - String[] conditions = new String[]{ - "lt(id, 45)", /* id between 23 and 45 */ - "not(lteq(id, 23))", /* id between 23 and 45 */ - "eq(first_name, Binary{\"alan\"})", /* first_name = 'alan' */ - "eq(last_name, Binary{\"smith\"})" /* 'smith' = last_name */ - }; - - FilterPredicate p = ParquetRecordReaderWrapper.toFilterPredicate(sarg); - String expected = String.format("and(and(and(%1$s, %2$s), %3$s), %4$s)", conditions); - assertEquals(expected, p.toString()); - - PredicateLeaf leaf = leaves.get(0); - assertEquals(PredicateLeaf.Type.INTEGER, leaf.getType()); - assertEquals(PredicateLeaf.Operator.BETWEEN, leaf.getOperator()); - assertEquals("id", leaf.getColumnName()); - assertEquals(null, leaf.getLiteral()); - assertEquals(23, leaf.getLiteralList().get(0)); - assertEquals(45, leaf.getLiteralList().get(1)); - - leaf = leaves.get(1); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator()); - assertEquals("first_name", leaf.getColumnName()); - assertEquals("alan", leaf.getLiteral()); - - leaf = leaves.get(2); - assertEquals(PredicateLeaf.Type.STRING, leaf.getType()); - assertEquals(PredicateLeaf.Operator.EQUALS, leaf.getOperator()); - assertEquals("last_name", leaf.getColumnName()); - assertEquals("smith", leaf.getLiteral()); - - assertEquals("(and leaf-0 leaf-1 leaf-2)", - sarg.getExpression().toString()); - assertNoSharedNodes(sarg.getExpression(), - Sets.newIdentityHashSet()); - } - - @Test - public void testExpression4() throws Exception { - /* id <> 12 and - first_name in ('john', 'sue') and - id in (34,50) */ - String exprStr = " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " id \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " int \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " 12 \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " boolean \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " first_name \n" + - " \n" + - " \n" + - " orc_people \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " string \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " john \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " sue \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - " \n" + - "