superset-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [incubator-superset] branch master updated: Workaround pandas bug in datetimes with time zones (#3910)
Date Mon, 20 Nov 2017 16:33:24 GMT
This is an automated email from the ASF dual-hosted git repository.

maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git


The following commit(s) were added to refs/heads/master by this push:
     new 4ae77ba  Workaround pandas bug in datetimes with time zones (#3910)
4ae77ba is described below

commit 4ae77ba8aff5d9289b28eb895ce644d99886b534
Author: bolkedebruin <bolkedebruin@users.noreply.github.com>
AuthorDate: Mon Nov 20 17:33:18 2017 +0100

    Workaround pandas bug in datetimes with time zones (#3910)
    
    A bug in to_dict(orient="records") in pandas/core/frame.py prevents
    datetimes with time zones to be worked with. This works around the
    issue in superset by re-implementing the logic of pandas in the
    correct way. Until pandas fixes the issue this code should stay.
    
    https://github.com/pandas-dev/pandas/issues/18372
    
    This closes #1929
---
 superset/dataframe.py |  6 +++++-
 tests/core_tests.py   | 21 ++++++++++++++++++++-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/superset/dataframe.py b/superset/dataframe.py
index 1d74fdd..cd9f95f 100644
--- a/superset/dataframe.py
+++ b/superset/dataframe.py
@@ -14,6 +14,7 @@ from datetime import date, datetime
 
 import numpy as np
 import pandas as pd
+from pandas.core.common import _maybe_box_datetimelike
 from pandas.core.dtypes.dtypes import ExtensionDtype
 from past.builtins import basestring
 
@@ -48,7 +49,10 @@ class SupersetDataFrame(object):
 
     @property
     def data(self):
-        return self.__df.to_dict(orient='records')
+        # work around for https://github.com/pandas-dev/pandas/issues/18372
+        return [dict((k, _maybe_box_datetimelike(v))
+                     for k, v in zip(self.__df.columns, np.atleast_1d(row)))
+                for row in self.__df.values]
 
     @classmethod
     def db_type(cls, dtype):
diff --git a/tests/core_tests.py b/tests/core_tests.py
index a821064..4b32c10 100644
--- a/tests/core_tests.py
+++ b/tests/core_tests.py
@@ -5,6 +5,7 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 import csv
+import datetime
 import doctest
 import io
 import json
@@ -13,9 +14,11 @@ import random
 import unittest
 
 from flask import escape
+import pandas as pd
+import psycopg2
 import sqlalchemy as sqla
 
-from superset import appbuilder, db, jinja_context, sm, sql_lab, utils
+from superset import appbuilder, dataframe, db, jinja_context, sm, sql_lab, utils
 from superset.connectors.sqla.models import SqlaTable
 from superset.models import core as models
 from superset.models.sql_lab import Query
@@ -786,6 +789,22 @@ class CoreTests(SupersetTestCase):
             {'name': ' NULL', 'sum__num': 0},
         )
 
+    def test_dataframe_timezone(self):
+        tz = psycopg2.tz.FixedOffsetTimezone(offset=60, name=None)
+        data = [(datetime.datetime(2017, 11, 18, 21, 53, 0, 219225, tzinfo=tz),),
+                (datetime.datetime(2017, 11, 18, 22, 6, 30, 61810, tzinfo=tz,),)]
+        df = dataframe.SupersetDataFrame(pd.DataFrame(data=list(data),
+                                                      columns=['data', ]))
+        data = df.data
+        self.assertDictEqual(
+            data[0],
+            {'data': pd.Timestamp('2017-11-18 21:53:00.219225+0100', tz=tz), },
+        )
+        self.assertDictEqual(
+            data[1],
+            {'data': pd.Timestamp('2017-11-18 22:06:30.061810+0100', tz=tz), },
+        )
+
 
 if __name__ == '__main__':
     unittest.main()

-- 
To stop receiving notification emails like this one, please contact
['"commits@superset.apache.org" <commits@superset.apache.org>'].

Mime
View raw message