kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [3/6] kudu git commit: KUDU-2399: Support IS NULL / IS NOT NULL predicates in Python
Date Mon, 30 Apr 2018 21:10:18 GMT
KUDU-2399: Support IS NULL / IS NOT NULL predicates in Python

Add is_null and is_not_null predicates in kudu-python. Had to change
the KuduTestBase and TestScanner to default to None and not replace
None with 'nothing' so I could write tests for null predicates.
I also fixed the test_scan_rows_string_predicate_and_projection test
since it was missing the projections and failing.

Change-Id: I52a22d6d8a8fe9a6049270596eb131799c37f82f
Reviewed-on: http://gerrit.cloudera.org:8080/10072
Tested-by: Kudu Jenkins
Reviewed-by: David Ribeiro Alves <davidralves@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3bf125f4
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3bf125f4
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3bf125f4

Branch: refs/heads/master
Commit: 3bf125f469a2929f7fe20e4e31f073376f37575d
Parents: 9173015
Author: Andy Stadtler <andy@phdata.io>
Authored: Sat Apr 14 14:06:29 2018 -0500
Committer: Grant Henke <granthenke@apache.org>
Committed: Mon Apr 30 19:58:11 2018 +0000

----------------------------------------------------------------------
 python/kudu/client.pyx            | 58 ++++++++++++++++++++++++++++++++++
 python/kudu/libkudu_client.pxd    |  2 ++
 python/kudu/tests/common.py       |  2 +-
 python/kudu/tests/test_scanner.py | 33 +++++++++++++++++--
 python/kudu/tests/util.py         |  8 +----
 5 files changed, 93 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/client.pyx
----------------------------------------------------------------------
diff --git a/python/kudu/client.pyx b/python/kudu/client.pyx
index 799b44e..5997bee 100644
--- a/python/kudu/client.pyx
+++ b/python/kudu/client.pyx
@@ -995,6 +995,64 @@ cdef class Column:
 
         return result
 
+    def is_not_null(Column self):
+        """
+        Creates a new IsNotNullPredicate for the Column which can be used for scanners
+        on this table.
+
+        Examples
+        --------
+        scanner.add_predicate(table[col_name].is_not_null())
+
+        Returns
+        -------
+        pred : Predicate
+        """
+
+        cdef:
+            KuduPredicate* pred
+            Slice col_name_slice
+            Predicate result
+            object _name = tobytes(self.name)
+
+        col_name_slice = Slice(<char*> _name, len(_name))
+
+        pred = (self.parent.ptr()
+                .NewIsNotNullPredicate(col_name_slice))
+
+        result = Predicate()
+        result.init(pred)
+
+        return result
+
+    def is_null(Column self):
+        """
+        Creates a new IsNullPredicate for the Column which can be used for scanners on this
table.
+
+        Examples
+        --------
+        scanner.add_predicate(table[col_name].is_null())
+
+        Returns
+        -------
+        pred : Predicate
+        """
+
+        cdef:
+            KuduPredicate* pred
+            Slice col_name_slice
+            Predicate result
+            object _name = tobytes(self.name)
+
+        col_name_slice = Slice(<char*> _name, len(_name))
+
+        pred = (self.parent.ptr()
+                .NewIsNullPredicate(col_name_slice))
+
+        result = Predicate()
+        result.init(pred)
+
+        return result
 
 class Partitioning(object):
     """ Argument to Client.create_table(...) to describe table partitioning. """

http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/libkudu_client.pxd
----------------------------------------------------------------------
diff --git a/python/kudu/libkudu_client.pxd b/python/kudu/libkudu_client.pxd
index a489784..5cd7731 100644
--- a/python/kudu/libkudu_client.pxd
+++ b/python/kudu/libkudu_client.pxd
@@ -609,6 +609,8 @@ cdef extern from "kudu/client/client.h" namespace "kudu::client" nogil:
                                               C_KuduValue* value);
         KuduPredicate* NewInListPredicate(const Slice& col_name,
                                           vector[C_KuduValue*]* values)
+        KuduPredicate* NewIsNotNullPredicate(const Slice& col_name)
+        KuduPredicate* NewIsNullPredicate(const Slice& col_name)
 
         KuduClient* client()
         # const PartitionSchema& partition_schema()

http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/tests/common.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/common.py b/python/kudu/tests/common.py
index 9596eaa..e1629ea 100644
--- a/python/kudu/tests/common.py
+++ b/python/kudu/tests/common.py
@@ -114,7 +114,7 @@ class KuduTestBase(object):
         builder = kudu.schema_builder()
         builder.add_column('key', kudu.int32, nullable=False)
         builder.add_column('int_val', kudu.int32)
-        builder.add_column('string_val', kudu.string, default='nothing')
+        builder.add_column('string_val', kudu.string, default=None)
         builder.add_column('unixtime_micros_val', kudu.unixtime_micros)
         builder.set_primary_keys(['key'])
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/tests/test_scanner.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py
index 00fe7a8..a4d0091 100644
--- a/python/kudu/tests/test_scanner.py
+++ b/python/kudu/tests/test_scanner.py
@@ -70,10 +70,9 @@ class TestScanner(TestScanBase):
 
     def test_scan_rows_string_predicate_and_projection(self):
         scanner = self.table.scanner()
-        scanner.set_projected_column_names(['key', 'string_val'])
-
         sv = self.table['string_val']
 
+        scanner.set_projected_column_indexes([0, 2])
         scanner.add_predicates([sv >= 'hello_20',
                                 sv <= 'hello_22'])
 
@@ -102,6 +101,36 @@ class TestScanner(TestScanBase):
 
         self.assertEqual(tuples, [self.tuples[98]])
 
+    def test_scan_rows_is_not_null_predicate(self):
+        """
+        Test scanner with an IsNotNull predicate on string_val column
+        """
+        pred = self.table['string_val'].is_not_null()
+        scanner = self.table.scanner()
+        scanner.add_predicate(pred)
+        scanner.open()
+
+        tuples = scanner.read_all_tuples()
+
+        rows = [i for i in range(100) if i % 2 == 0]
+
+        self.assertEqual(sorted(tuples), [self.tuples[i] for i in rows])
+
+    def test_scan_rows_is_null_predicate(self):
+        """
+        Test scanner with an IsNull predicate on string_val column
+        """
+        pred = self.table['string_val'].is_null()
+        scanner = self.table.scanner()
+        scanner.add_predicate(pred)
+        scanner.open()
+
+        tuples = scanner.read_all_tuples()
+
+        rows = [i for i in range(100) if i % 2 != 0]
+
+        self.assertEqual(sorted(tuples), [self.tuples[i] for i in rows])
+
     def test_index_projection_with_schema(self):
         scanner = self.table.scanner()
         scanner.set_projected_column_indexes([0, 1])

http://git-wip-us.apache.org/repos/asf/kudu/blob/3bf125f4/python/kudu/tests/util.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/util.py b/python/kudu/tests/util.py
index a6779fc..6823b3a 100644
--- a/python/kudu/tests/util.py
+++ b/python/kudu/tests/util.py
@@ -56,13 +56,7 @@ class TestScanBase(KuduTestBase, unittest.TestCase):
         session.flush()
 
         self.table = table
-        self.tuples = []
-
-        # Replace missing values w/ defaults to test default values.
-        for tuple in tuples:
-            if tuple[2] == None:
-                tuple = (tuple[0], tuple[1], 'nothing', tuple[3])
-            self.tuples.append(tuple)
+        self.tuples = tuples
 
         # Create table to test all types
         # for various predicate tests


Mime
View raw message