kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject kudu git commit: Remove Pandas as a Python Client Requirement
Date Thu, 04 Oct 2018 18:17:45 GMT
Repository: kudu
Updated Branches:
  refs/heads/master da28d0aee -> 131c5d7af


Remove Pandas as a Python Client Requirement

This is to remove the pandas dependency for users.
We will put the pandas dependency in the jenkins script
build-and-test.sh as a workaround. This is based on the initial
work done here https://gerrit.cloudera.org/#/c/10809/4

Change-Id: I2cce7d529a804e712ccc32fb2774f5d0ee9dbdf8
Reviewed-on: http://gerrit.cloudera.org:8080/11557
Tested-by: Kudu Jenkins
Reviewed-by: Jordan Birdsell <jtbirdsell@apache.org>
Reviewed-by: Adar Dembo <adar@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/131c5d7a
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/131c5d7a
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/131c5d7a

Branch: refs/heads/master
Commit: 131c5d7af0a7c4b398ef01aaaea8378f18f84014
Parents: da28d0a
Author: Andy Stadtler <andy@phdata.io>
Authored: Mon Oct 1 16:13:36 2018 -0500
Committer: Adar Dembo <adar@cloudera.com>
Committed: Thu Oct 4 18:17:25 2018 +0000

----------------------------------------------------------------------
 build-support/jenkins/build-and-test.sh | 12 ++++++++++++
 python/kudu/tests/test_scanner.py       | 11 +++++++++--
 python/requirements.txt                 |  6 ------
 python/setup.py                         |  6 ++----
 4 files changed, 23 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index df0336c..6ba91c9 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -503,6 +503,14 @@ if [ "$BUILD_PYTHON" == "1" ]; then
   # pass in the current values of CC and CXX.
   CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt
 
+  # We need to install Pandas manually since its not a required package but is needed
+  # to run all of the tests.
+  # pandas 0.18 dropped support for python 2.6.
+  #
+  # See https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016
+  # for more details.
+  pip install $PIP_INSTALL_FLAGS "pandas<0.18"
+
   # Delete old Cython extensions to force them to be rebuilt.
   rm -Rf build kudu_python.egg-info kudu/*.so
 
@@ -561,6 +569,10 @@ if [ "$BUILD_PYTHON3" == "1" ]; then
   # pass in the current values of CC and CXX.
   CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt
 
+  # We need to install Pandas manually since its not a required package but is needed
+  # to run all of the tests.
+  pip install $PIP_INSTALL_FLAGS pandas
+
   # Delete old Cython extensions to force them to be rebuilt.
   rm -Rf build kudu_python.egg-info kudu/*.so
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/python/kudu/tests/test_scanner.py
----------------------------------------------------------------------
diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py
index 6d23d96..3c7fe57 100644
--- a/python/kudu/tests/test_scanner.py
+++ b/python/kudu/tests/test_scanner.py
@@ -328,6 +328,8 @@ class TestScanner(TestScanBase):
             self.assertEqual(sorted(scanner.read_all_tuples()),
                              sorted(self.tuples))
 
+    @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS),
+                        reason="Pandas required to run this test.")
     def test_scanner_to_pandas_types(self):
         """
         This test confirms that data types are converted as expected to Pandas.
@@ -357,6 +359,8 @@ class TestScanner(TestScanBase):
             self.assertEqual(types[6], np.object)
             self.assertEqual(types[7], np.float32)
 
+    @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS),
+                        reason="Pandas required to run this test.")
     def test_scanner_to_pandas_row_count(self):
         """
         This test confirms that the record counts match between Pandas and the scanner.
@@ -367,6 +371,8 @@ class TestScanner(TestScanBase):
         df = scanner.to_pandas()
         self.assertEqual(scanner_count, df.shape[0])
 
+    @pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS),
+                        reason="Pandas required to run this test.")
     def test_scanner_to_pandas_index(self):
         """
         This test confirms that an index is correctly applied.
@@ -376,8 +382,9 @@ class TestScanner(TestScanBase):
         self.assertEqual(df.index.name, 'key')
         self.assertEqual(list(df.index), [1, 2])
 
-    @pytest.mark.skipif(not(kudu.CLIENT_SUPPORTS_DECIMAL),
-                        reason="Decimal is not supported on this client.")
+    @pytest.mark.skipif((not(kudu.CLIENT_SUPPORTS_PANDAS) and
+                        (not(kudu.CLIENT_SUPPORTS_DECIMAL))),
+                        reason="Pandas and Decimal support required to run this test.")
     def test_scanner_to_pandas_index(self):
         """
         This test confirms that a decimal column is coerced to a double when specified.

http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/python/requirements.txt
----------------------------------------------------------------------
diff --git a/python/requirements.txt b/python/requirements.txt
index eeb91af..5b6cbc1 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -8,12 +8,6 @@ pytest >=2.8,<3.3
 # See https://pypi.org/project/pytest-timeout/#id5 for more details.
 pytest-timeout >=1.1.0,<1.2.1
 
-# pandas 0.18 dropped support for python 2.6.
-#
-# See https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016
-# for more details.
-pandas <0.18
-
 cython ==0.26.1
 six
 unittest2

http://git-wip-us.apache.org/repos/asf/kudu/blob/131c5d7a/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 330f253..ef0013a 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -194,15 +194,13 @@ setup(
     # requirements.txt so that dependencies aren't downloaded at test-time
     # (when it's more difficult to override various pip installation options).
     #
-    # pytest 3.3 [1], pytest-timeout 1.2.1 [2], and pandas 0.18 [3] dropped
+    # pytest 3.3 [1] and pytest-timeout 1.2.1 [2] dropped
     # support for python 2.6.
     #
     # 1. https://docs.pytest.org/en/latest/changelog.html#id164
     # 2. https://pypi.org/project/pytest-timeout/#id5
-    # 3. https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016
     tests_require=['pytest >=2.8,<3.3',
-                   'pytest-timeout >=1.1.0,<1.2.1',
-                   'pandas <0.18'],
+                   'pytest-timeout >=1.1.0,<1.2.1'],
 
     install_requires=['cython >= 0.21', 'pytz', 'six'],
     description=DESCRIPTION,


Mime
View raw message