hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From goden...@apache.org
Subject incubator-hawq git commit: HAWQ-779 support pxf filter pushdwon at the 'CREATE PLAN' stage , and more filter type & op
Date Wed, 14 Sep 2016 19:31:25 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master cd81289bc -> 6416b536d


HAWQ-779 support pxf filter pushdwon at the 'CREATE PLAN' stage ,and more filter type &
op


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6416b536
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6416b536
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6416b536

Branch: refs/heads/master
Commit: 6416b536d273b5add89482d50ea403c70572da80
Parents: cd81289
Author: Devin Jia <jiadx@inspur.com>
Authored: Fri Aug 5 15:05:50 2016 +0800
Committer: Goden Yao <godenyao@apache.org>
Committed: Wed Sep 14 12:30:25 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hawq/pxf/api/FilterParser.java   |  4 +++-
 .../org/apache/hawq/pxf/api/FilterParserTest.java    |  8 ++++++++
 .../hawq/pxf/plugins/hbase/HBaseFilterBuilder.java   |  8 ++++++++
 .../hawq/pxf/plugins/hive/HiveDataFragmenter.java    | 12 ++++++++++++
 src/backend/access/external/pxffilters.c             | 15 +++++++++++++--
 src/backend/access/external/test/pxffilters_test.c   | 14 +++++++++++++-
 src/backend/optimizer/plan/createplan.c              |  8 +++++++-
 src/include/access/pxffilters.h                      |  3 ++-
 8 files changed, 66 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
index ae96d22..00fbf2b 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/FilterParser.java
@@ -61,7 +61,8 @@ public class FilterParser {
         HDOP_GE,
         HDOP_EQ,
         HDOP_NE,
-        HDOP_AND
+        HDOP_AND,
+        HDOP_LIKE
     }
 
     /**
@@ -392,6 +393,7 @@ public class FilterParser {
         operatorTranslationMap.put(5, Operation.HDOP_EQ);
         operatorTranslationMap.put(6, Operation.HDOP_NE);
         operatorTranslationMap.put(7, Operation.HDOP_AND);
+        operatorTranslationMap.put(8, Operation.HDOP_LIKE);
         return operatorTranslationMap;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
index 0308f7f..1ded4a3 100644
--- a/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
+++ b/pxf/pxf-api/src/test/java/org/apache/hawq/pxf/api/FilterParserTest.java
@@ -215,6 +215,10 @@ public class FilterParserTest {
         filter = "a1c2o7";
         op = Operation.HDOP_AND;
         runParseOneOperation("this filter was build from HDOP_AND", filter, op);
+        
+        filter = "a1c2o8";
+        op = Operation.HDOP_LIKE;
+        runParseOneOperation("this filter was built from HDOP_LIKE", filter, op);
     }
 
     @Test
@@ -247,6 +251,10 @@ public class FilterParserTest {
         filter = "c2a1o7";
         op = Operation.HDOP_AND;
         runParseOneOperation("this filter was build from HDOP_AND using reverse!", filter,
op);
+        
+        filter = "c2a1o8";
+        op = Operation.HDOP_LIKE;
+        runParseOneOperation("this filter was build from HDOP_LIKE using reverse!", filter,
op);
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
index 6dc271e..8eadc88 100644
--- a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
+++ b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/HBaseFilterBuilder.java
@@ -165,6 +165,14 @@ public class HBaseFilterBuilder implements FilterParser.FilterBuilder
{
         ByteArrayComparable comparator = getComparator(hbaseColumn.columnTypeCode(),
                 constant.constant());
 
+        if(operatorsMap.get(opId) == null){
+            //HBase does not support HDOP_LIKE, use 'NOT NULL' comparator
+            return new SingleColumnValueFilter(hbaseColumn.columnFamilyBytes(),
+                    hbaseColumn.qualifierBytes(),
+                    CompareFilter.CompareOp.NOT_EQUAL,
+                    new NullComparator());
+        }
+
         /**
          * If row key is of type TEXT, allow filter in start/stop row key API in
          * HBaseAccessor/Scan object.

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
index ded9627..2fe31cd 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveDataFragmenter.java
@@ -25,6 +25,8 @@ import java.util.ListIterator;
 import java.util.Properties;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.Map;
+import java.util.HashMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -37,6 +39,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputFormat;
@@ -48,6 +51,7 @@ import org.apache.hawq.pxf.api.Fragment;
 import org.apache.hawq.pxf.api.Fragmenter;
 import org.apache.hawq.pxf.api.FragmentsStats;
 import org.apache.hawq.pxf.api.Metadata;
+import org.apache.hawq.pxf.api.io.DataType;
 import org.apache.hawq.pxf.api.utilities.ColumnDescriptor;
 import org.apache.hawq.pxf.api.utilities.InputData;
 import org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities;
@@ -87,6 +91,7 @@ public class HiveDataFragmenter extends Fragmenter {
     // partition filtering
     private Set<String> setPartitions = new TreeSet<String>(
             String.CASE_INSENSITIVE_ORDER);
+    private Map<String, String> partitionkeyTypes = new HashMap<>();
 
     /**
      * A Hive table unit - means a subset of the HIVE table, where we can say
@@ -191,6 +196,7 @@ public class HiveDataFragmenter extends Fragmenter {
             // Save all hive partition names in a set for later filter match
             for (FieldSchema fs : tbl.getPartitionKeys()) {
                 setPartitions.add(fs.getName());
+				partitionkeyTypes.put(fs.getName(), fs.getType());
             }
 
             LOG.debug("setPartitions :" + setPartitions);
@@ -451,6 +457,12 @@ public class HiveDataFragmenter extends Fragmenter {
             return false;
         }
 
+		if (!partitionkeyTypes.get(filterColumnName).equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME))
{
+            LOG.debug("Filter type is not string type , ignore this filter for hive: "
+                    + filter);
+            return false;
+        }
+
         filtersString.append(prefix);
         filtersString.append(filterColumnName);
         filtersString.append(HIVE_API_EQ);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/access/external/pxffilters.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/pxffilters.c b/src/backend/access/external/pxffilters.c
index 405e6d7..cdb2cb6 100644
--- a/src/backend/access/external/pxffilters.c
+++ b/src/backend/access/external/pxffilters.c
@@ -79,6 +79,7 @@ dbop_pxfop_map pxf_supported_opr[] =
 	{665 /* text_le */, PXFOP_LE},
 	{667 /* text_ge */, PXFOP_GE},
 	{531 /* textlt  */, PXFOP_NE},
+	{1209 /* textlike  */, PXFOP_LIKE},
 
 	/* int2 to int4 */
 	{Int24EqualOperator /* int24eq */, PXFOP_EQ},
@@ -126,7 +127,15 @@ dbop_pxfop_map pxf_supported_opr[] =
 	{1871 /* int82gt */, PXFOP_GT},
 	{1872 /* int82le */, PXFOP_LE},
 	{1873 /* int82ge */, PXFOP_GE},
-	{1869 /* int82ne */, PXFOP_NE}
+	{1869 /* int82ne */, PXFOP_NE},
+
+	/* date */
+	{DateEqualOperator  /* eq */, PXFOP_EQ},
+	{1095  /* date_lt */, PXFOP_LT},
+	{1097 /* date_gt */, PXFOP_GT},
+	{1096 /* date_le */, PXFOP_LE},
+	{1098 /* date_ge */, PXFOP_GE},
+	{1094 /* date_ne */, PXFOP_NE}
 
 };
 
@@ -143,7 +152,8 @@ Oid pxf_supported_types[] =
 	BPCHAROID,
 	CHAROID,
 	BYTEAOID,
-	BOOLOID
+	BOOLOID,
+	DATEOID
 };
 
 /*
@@ -559,6 +569,7 @@ const_to_str(Const *constval, StringInfo buf)
 		case BPCHAROID:
 		case CHAROID:
 		case BYTEAOID:
+		case DATEOID:
 			appendStringInfo(buf, "\\\"%s\\\"", extval);
 			break;
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/access/external/test/pxffilters_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxffilters_test.c b/src/backend/access/external/test/pxffilters_test.c
index a5b3fb0..65b1aef 100644
--- a/src/backend/access/external/test/pxffilters_test.c
+++ b/src/backend/access/external/test/pxffilters_test.c
@@ -42,6 +42,7 @@ test__supported_filter_type(void **state)
 		CHAROID,
 		BYTEAOID,
 		BOOLOID,
+		DATEOID,
 		CIRCLEOID /* unsupported type */
 	};
 
@@ -61,7 +62,7 @@ test__supported_filter_type(void **state)
 
 	/* go over pxf_supported_types array */
 	int nargs = sizeof(pxf_supported_types) / sizeof(Oid);
-	assert_int_equal(nargs, 12);
+	assert_int_equal(nargs, 13);
 	for (i = 0; i < nargs; ++i)
 	{
 		assert_true(supported_filter_type(pxf_supported_types[i]));
@@ -184,6 +185,7 @@ test__const_to_str__text(void **state)
 	verify__const_to_str(false, "isn't", BPCHAROID, "\\\"isn't\\\"");
 	verify__const_to_str(false, "funny", CHAROID, "\\\"funny\\\"");
 	verify__const_to_str(false, "anymore", BYTEAOID, "\\\"anymore\\\"");
+	verify__const_to_str(false, "iamdate", DATEOID, "\\\"iamdate\\\"");
 }
 
 void
@@ -523,6 +525,10 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
 			PXF_ATTR_CODE, 5, NULL,
 			PXF_CONST_CODE, 0, "\"Winston\"",
 			PXFOP_GE);
+	PxfFilterDesc* filter5 = build_filter(
+			PXF_ATTR_CODE, 6, NULL,
+			PXF_CONST_CODE, 0, "\"Eric-%\"",
+			PXFOP_LIKE);
 
 	filter_list = lappend(filter_list, filter1);
 	filter_list = lappend(filter_list, filter2);
@@ -543,6 +549,12 @@ test__pxf_serialize_filter_list__manyFilters(void **state)
 	assert_string_equal(result, "a1c1983o2a2c1985o1o7a3c\"George Orwell\"o5o7a4c\"Winston\"o4o7");
 	pfree(result);
 
+	filter_list = lappend(filter_list, filter5);
+
+	result = pxf_serialize_filter_list(filter_list);
+	assert_string_equal(result, "a1c1983o2a2c1985o1o7a3c\"George Orwell\"o5o7a4c\"Winston\"o4o7a5c\"Eric-%\"o8o7");
+	pfree(result);
+
 	pxf_free_filter_list(filter_list);
 	filter_list = NIL;
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/backend/optimizer/plan/createplan.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index b5a5cea..d7efa62 100644
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -1144,9 +1144,15 @@ static char** create_pxf_plan(char **segdb_file_map, RelOptInfo *rel,
int total_
 	
 	
 	Relation relation = RelationIdGetRelation(planner_rt_fetch(scan_relid, ctx->root)->relid);
-	segdb_work_map = map_hddata_2gp_segments(uri_str, 
+	if (pxf_enable_filter_pushdown){
+		segdb_work_map = map_hddata_2gp_segments(uri_str,
+											 total_segs, segs_participating,
+											 relation, ctx->root->parse->jointree->quals);
+	}else{
+		segdb_work_map = map_hddata_2gp_segments(uri_str,
 											 total_segs, segs_participating,
 											 relation, NULL);
+	}
 	Assert(segdb_work_map != NULL);
 	RelationClose(relation);
 	

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6416b536/src/include/access/pxffilters.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxffilters.h b/src/include/access/pxffilters.h
index 1409e39..f54c47c 100644
--- a/src/include/access/pxffilters.h
+++ b/src/include/access/pxffilters.h
@@ -44,7 +44,8 @@ typedef enum PxfOperatorCode
 	PXFOP_GE,
 	PXFOP_EQ,
 	PXFOP_NE,
-	PXFOP_AND
+	PXFOP_AND,
+	PXFOP_LIKE
 
 } PxfOperatorCode;
 


Mime
View raw message