superset-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [incubator-superset] branch master updated: chore: taking db dependencies out of requirements-dev.txt (#7605)
Date Mon, 24 Jun 2019 05:37:49 GMT
This is an automated email from the ASF dual-hosted git repository.

maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git


The following commit(s) were added to refs/heads/master by this push:
     new 859d6e7  chore: taking db dependencies out of requirements-dev.txt (#7605)
859d6e7 is described below

commit 859d6e7c6a8b89c7b3fd0687aaca51bd2868144a
Author: Maxime Beauchemin <maximebeauchemin@gmail.com>
AuthorDate: Sun Jun 23 22:37:41 2019 -0700

    chore: taking db dependencies out of requirements-dev.txt (#7605)
    
    * chore: taking db dependencies out of requirements-dev.txt
    
    The deps on mysqlclient and psycopg2
    
    * Fix unit tests
    
    * fix tox.ini
    
    * fix tests
---
 .gitignore                        |  1 +
 requirements-dev.txt              |  2 --
 setup.py                          |  5 ++++-
 superset/db_engine_specs/mysql.py |  2 +-
 superset/db_engines/hive.py       |  6 +++---
 superset/sql_parse.py             |  2 +-
 tests/base_tests.py               |  9 +++++++++
 tests/db_engine_specs_test.py     | 37 ++++++++++++++++++-------------------
 tests/model_tests.py              |  9 +++++++++
 tox.ini                           | 12 ++++++++++++
 10 files changed, 58 insertions(+), 27 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2c553ad..fcfe9cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,7 @@ changelog.sh
 dist
 dump.rdb
 env
+venv*
 env_py3
 envpy3
 env36
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 7059084..1b008ad 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -23,7 +23,6 @@ flake8==3.7.7
 flask-cors==3.0.7
 ipdb==0.12
 mypy==0.670
-mysqlclient==1.4.2.post1
 nose==1.3.7
 pip-tools==3.7.0
 psycopg2-binary==2.7.5
@@ -33,5 +32,4 @@ pylint==1.9.2
 python-dotenv==0.10.1
 redis==2.10.6
 statsd==3.3.0
-thrift==0.11.0
 tox==3.11.1
diff --git a/setup.py b/setup.py
index 28c6bbd..8d3fcc6 100644
--- a/setup.py
+++ b/setup.py
@@ -113,12 +113,15 @@ setup(
             'pandas_gbq>=0.10.0',
         ],
         'cors': ['flask-cors>=2.0.0'],
+        'gsheets': ['gsheetsdb>=0.1.9'],
         'hive': [
             'pyhive[hive]>=0.6.1',
             'tableschema',
+            'thrift>=0.11.0, <1.0.0',
         ],
+        'mysql': ['mysqlclient==1.4.2.post1'],
+        'postgres': ['psycopg2-binary==2.7.5'],
         'presto': ['pyhive[presto]>=0.4.0'],
-        'gsheets': ['gsheetsdb>=0.1.9'],
     },
     author='Apache Software Foundation',
     author_email='dev@superset.incubator.apache.org',
diff --git a/superset/db_engine_specs/mysql.py b/superset/db_engine_specs/mysql.py
index 31a6f4c..5019407 100644
--- a/superset/db_engine_specs/mysql.py
+++ b/superset/db_engine_specs/mysql.py
@@ -67,7 +67,7 @@ class MySQLEngineSpec(BaseEngineSpec):
     def get_datatype(cls, type_code):
         if not cls.type_code_map:
             # only import and store if needed at least once
-            import MySQLdb
+            import MySQLdb  # pylint: disable=import-error
             ft = MySQLdb.constants.FIELD_TYPE
             cls.type_code_map = {
                 getattr(ft, k): k
diff --git a/superset/db_engines/hive.py b/superset/db_engines/hive.py
index 6334257..bab2e38 100644
--- a/superset/db_engines/hive.py
+++ b/superset/db_engines/hive.py
@@ -28,9 +28,9 @@ def fetch_logs(self, max_rows=1024,
     .. note::
         This is not a part of DB-API.
     """
-    from pyhive import hive
-    from TCLIService import ttypes
-    from thrift import Thrift
+    from pyhive import hive  # noqa
+    from TCLIService import ttypes # noqa
+    from thrift import Thrift  # pylint: disable=import-error
     orientation = orientation or ttypes.TFetchOrientation.FETCH_NEXT
     try:
         req = ttypes.TGetLogReq(operationHandle=self._operationHandle)
diff --git a/superset/sql_parse.py b/superset/sql_parse.py
index e2c25ef..868800b 100644
--- a/superset/sql_parse.py
+++ b/superset/sql_parse.py
@@ -193,7 +193,7 @@ class ParsedQuery(object):
         """returns the query with the specified limit"""
         """does not change the underlying query"""
         if not self._limit:
-            return f'{self.sql}\nLIMIT {new_limit}'
+            return f'{self.stripped()}\nLIMIT {new_limit}'
         limit_pos = None
         tokens = self._parsed[0].tokens
         # Add all items to before_str until there is a limit
diff --git a/tests/base_tests.py b/tests/base_tests.py
index b24193c..7ccb637 100644
--- a/tests/base_tests.py
+++ b/tests/base_tests.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Unit tests for Superset"""
+import imp
 import json
 import unittest
 from unittest.mock import Mock, patch
@@ -73,6 +74,14 @@ class SupersetTestCase(unittest.TestCase):
             .one()
         )
 
+    @staticmethod
+    def is_module_installed(module_name):
+        try:
+            imp.find_module(module_name)
+            return True
+        except ImportError:
+            return False
+
     def get_or_create(self, cls, criteria, session, **kwargs):
         obj = session.query(cls).filter_by(**criteria).first()
         if not obj:
diff --git a/tests/db_engine_specs_test.py b/tests/db_engine_specs_test.py
index 6acc8ba..e3e81bb 100644
--- a/tests/db_engine_specs_test.py
+++ b/tests/db_engine_specs_test.py
@@ -14,6 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import unittest
 from unittest import mock
 
 from sqlalchemy import column, literal_column, select, table
@@ -137,7 +138,7 @@ class DbEngineSpecsTestCase(SupersetTestCase):
             HiveEngineSpec.extract_error_message(Exception(msg)))
 
     def get_generic_database(self):
-        return Database(sqlalchemy_uri='mysql://localhost')
+        return Database(sqlalchemy_uri='sqlite://')
 
     def sql_limit_regex(
             self, sql, expected_sql,
@@ -177,22 +178,16 @@ class DbEngineSpecsTestCase(SupersetTestCase):
     def test_wrapped_query(self):
         self.sql_limit_regex(
             'SELECT * FROM a',
-            'SELECT * \nFROM (SELECT * FROM a) AS inner_qry \n LIMIT 1000',
-            MssqlEngineSpec,
-        )
-
-    def test_wrapped_semi(self):
-        self.sql_limit_regex(
-            'SELECT * FROM a;',
-            'SELECT * \nFROM (SELECT * FROM a) AS inner_qry \n LIMIT 1000',
+            'SELECT * \nFROM (SELECT * FROM a) AS inner_qry\n LIMIT 1000 OFFSET 0',
             MssqlEngineSpec,
         )
 
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed('MySQLdb'), 'mysqlclient not installed')
     def test_wrapped_semi_tabs(self):
         self.sql_limit_regex(
             'SELECT * FROM a  \t \n   ; \t  \n  ',
-            'SELECT * \nFROM (SELECT * FROM a) AS inner_qry \n LIMIT 1000',
-            MssqlEngineSpec,
+            'SELECT * FROM a\nLIMIT 1000',
         )
 
     def test_simple_limit_query(self):
@@ -247,10 +242,18 @@ class DbEngineSpecsTestCase(SupersetTestCase):
                 LIMIT         1000""",
         )
 
-    def test_get_datatype(self):
-        self.assertEquals('STRING', PrestoEngineSpec.get_datatype('string'))
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed('MySQLdb'), 'mysqlclient not installed')
+    def test_get_datatype_mysql(self):
         self.assertEquals('TINY', MySQLEngineSpec.get_datatype(1))
         self.assertEquals('VARCHAR', MySQLEngineSpec.get_datatype(15))
+
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed('pyhive'), 'pyhive not installed')
+    def test_get_datatype_presto(self):
+        self.assertEquals('STRING', PrestoEngineSpec.get_datatype('string'))
+
+    def test_get_datatype(self):
         self.assertEquals('VARCHAR', BaseEngineSpec.get_datatype('VARCHAR'))
 
     def test_limit_with_implicit_offset(self):
@@ -291,12 +294,8 @@ class DbEngineSpecsTestCase(SupersetTestCase):
 
     def test_limit_with_non_token_limit(self):
         self.sql_limit_regex(
-            """
-                SELECT
-                    'LIMIT 777'""",
-            """
-                SELECT
-                    'LIMIT 777'\nLIMIT 1000""",
+            """SELECT 'LIMIT 777'""",
+            """SELECT 'LIMIT 777'\nLIMIT 1000""",
         )
 
     def test_time_grain_blacklist(self):
diff --git a/tests/model_tests.py b/tests/model_tests.py
index 53e53cc..578dd78 100644
--- a/tests/model_tests.py
+++ b/tests/model_tests.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 import textwrap
+import unittest
 
 import pandas
 from sqlalchemy.engine.url import make_url
@@ -56,6 +57,10 @@ class DatabaseModelTestCase(SupersetTestCase):
         db = make_url(model.get_sqla_engine(schema='foo').url).database
         self.assertEquals('prod', db)
 
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed('thrift'), 'thrift not installed')
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed('pyhive'), 'pyhive not installed')
     def test_database_schema_hive(self):
         sqlalchemy_uri = 'hive://hive@hive.airbnb.io:10000/default?auth=NOSASL'
         model = Database(sqlalchemy_uri=sqlalchemy_uri)
@@ -65,6 +70,8 @@ class DatabaseModelTestCase(SupersetTestCase):
         db = make_url(model.get_sqla_engine(schema='core_db').url).database
         self.assertEquals('core_db', db)
 
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed('MySQLdb'), 'mysqlclient not installed')
     def test_database_schema_mysql(self):
         sqlalchemy_uri = 'mysql://root@localhost/superset'
         model = Database(sqlalchemy_uri=sqlalchemy_uri)
@@ -75,6 +82,8 @@ class DatabaseModelTestCase(SupersetTestCase):
         db = make_url(model.get_sqla_engine(schema='staging').url).database
         self.assertEquals('staging', db)
 
+    @unittest.skipUnless(
+        SupersetTestCase.is_module_installed('MySQLdb'), 'mysqlclient not installed')
     def test_database_impersonate_user(self):
         uri = 'mysql://root@localhost'
         example_user = 'giuseppe'
diff --git a/tox.ini b/tox.ini
index 8b39058..dc8f40d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -64,6 +64,18 @@ setenv =
 whitelist_externals =
     npm
 
+[testenv:py36-mysql]
+deps =
+    -rrequirements.txt
+    -rrequirements-dev.txt
+    .[mysql]
+
+[testenv:py36-postgres]
+deps =
+    -rrequirements.txt
+    -rrequirements-dev.txt
+    .[postgres]
+
 [testenv:cypress-dashboard]
 commands =
     npm install -g npm@'>=6.5.0'


Mime
View raw message