superset-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [incubator-superset] branch master updated: fix: SqlaColumn.type overflow on mysql (#7606)
Date Mon, 03 Jun 2019 16:52:38 GMT
This is an automated email from the ASF dual-hosted git repository.

maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git


The following commit(s) were added to refs/heads/master by this push:
     new 45b9880  fix: SqlaColumn.type overflow on mysql (#7606)
45b9880 is described below

commit 45b988051614bda145add7934af2d310f3a94754
Author: Maxime Beauchemin <maximebeauchemin@gmail.com>
AuthorDate: Mon Jun 3 09:52:25 2019 -0700

    fix: SqlaColumn.type overflow on mysql (#7606)
    
    * fix: SqlaColumn.type overflow on mysql
    
    Hitting a new error when loading examples around a datatype that exceeds
    the 32 chars limit on SqlaColumn.type
    
    Type includes the 'COLLATE utf8mb4_general_ci' suffix which is too
    verbose and not needed in that context.
    
    * fix tests
---
 superset/connectors/sqla/models.py |  2 +-
 superset/db_engine_specs.py        | 15 +++++++++++++++
 tests/base_tests.py                |  7 +++++--
 tests/db_engine_specs_test.py      | 14 ++++++++++++++
 4 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py
index de9f4d1..138b0e5 100644
--- a/superset/connectors/sqla/models.py
+++ b/superset/connectors/sqla/models.py
@@ -900,7 +900,7 @@ class SqlaTable(Model, BaseDatasource):
 
         for col in table.columns:
             try:
-                datatype = col.type.compile(dialect=db_dialect).upper()
+                datatype = db_engine_spec.column_datatype_to_string(col.type, db_dialect)
             except Exception as e:
                 datatype = 'UNKNOWN'
                 logging.error(
diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py
index 5136bca..922e7cc 100644
--- a/superset/db_engine_specs.py
+++ b/superset/db_engine_specs.py
@@ -529,6 +529,10 @@ class BaseEngineSpec(object):
             label = label[:cls.max_column_name_length]
         return label
 
+    @classmethod
+    def column_datatype_to_string(cls, sqla_column_type, dialect):
+        return sqla_column_type.compile(dialect=dialect).upper()
+
 
 class PostgresBaseEngineSpec(BaseEngineSpec):
     """ Abstract class for Postgres 'like' databases """
@@ -864,6 +868,17 @@ class MySQLEngineSpec(BaseEngineSpec):
             pass
         return message
 
+    @classmethod
+    def column_datatype_to_string(cls, sqla_column_type, dialect):
+        datatype = super().column_datatype_to_string(sqla_column_type, dialect)
+        # MySQL dialect started returning long overflowing datatype
+        # as in 'VARCHAR(255) COLLATE UTF8MB4_GENERAL_CI'
+        # and we don't need the verbose collation type
+        str_cutoff = ' COLLATE '
+        if str_cutoff in datatype:
+            datatype = datatype.split(str_cutoff)[0]
+        return datatype
+
 
 class PrestoEngineSpec(BaseEngineSpec):
     engine = 'presto'
diff --git a/tests/base_tests.py b/tests/base_tests.py
index 6de082a..b24193c 100644
--- a/tests/base_tests.py
+++ b/tests/base_tests.py
@@ -174,12 +174,15 @@ class SupersetTestCase(unittest.TestCase):
                     perm.view_menu and table.perm in perm.view_menu.name):
                 security_manager.del_permission_role(public_role, perm)
 
+    def get_main_database(self):
+        return get_main_database(db.session)
+
     def run_sql(self, sql, client_id=None, user_name=None, raise_on_error=False,
                 query_limit=None):
         if user_name:
             self.logout()
             self.login(username=(user_name if user_name else 'admin'))
-        dbid = get_main_database(db.session).id
+        dbid = self.get_main_database().id
         resp = self.get_json_resp(
             '/superset/sql_json/',
             raise_on_error=False,
@@ -195,7 +198,7 @@ class SupersetTestCase(unittest.TestCase):
         if user_name:
             self.logout()
             self.login(username=(user_name if user_name else 'admin'))
-        dbid = get_main_database(db.session).id
+        dbid = self.get_main_database().id
         resp = self.get_json_resp(
             '/superset/validate_sql_json/',
             raise_on_error=False,
diff --git a/tests/db_engine_specs_test.py b/tests/db_engine_specs_test.py
index 4491914..aae7380 100644
--- a/tests/db_engine_specs_test.py
+++ b/tests/db_engine_specs_test.py
@@ -814,3 +814,17 @@ class DbEngineSpecsTestCase(SupersetTestCase):
         expr = PinotEngineSpec.get_timestamp_expr(col, 'epoch_s', 'P1M')
         result = str(expr.compile())
         self.assertEqual(result, 'DATETIMECONVERT(tstamp, "1:SECONDS:EPOCH", "1:SECONDS:EPOCH",
"1:MONTHS")')  # noqa
+
+    def test_column_datatype_to_string(self):
+        main_db = self.get_main_database()
+        sqla_table = main_db.get_table('energy_usage')
+        dialect = main_db.get_dialect()
+        col_names = [
+            main_db.db_engine_spec.column_datatype_to_string(c.type, dialect)
+            for c in sqla_table.columns
+        ]
+        if main_db.backend == 'postgresql':
+            expected = ['VARCHAR(255)', 'VARCHAR(255)', 'DOUBLE PRECISION']
+        else:
+            expected = ['VARCHAR(255)', 'VARCHAR(255)', 'FLOAT']
+        self.assertEquals(col_names, expected)


Mime
View raw message