superset-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maximebeauche...@apache.org
Subject [incubator-superset] branch master updated: Summary: Introduce configs for default relative end time and the epoch-tz (#6721)
Date Sat, 02 Feb 2019 00:34:18 GMT
This is an automated email from the ASF dual-hosted git repository.

maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git


The following commit(s) were added to refs/heads/master by this push:
     new cecbba3  Summary: Introduce configs for default relative end time and the epoch-tz
(#6721)
cecbba3 is described below

commit cecbba3fe63cfb6ea294cec90f57e5282b6dfb6c
Author: agrawaldevesh <dagrawal@uber.com>
AuthorDate: Fri Feb 1 16:34:13 2019 -0800

    Summary: Introduce configs for default relative end time and the epoch-tz (#6721)
    
    Introduce a config DEFAULT_RELATIVE_END_TIME which is used when
    computing the "Last X days". The default behavior (as currently) is to
    let that be 'today', which actually means the 0th second of today. We
    can also let it be 'now' which means the data as of now (ie at query
    time).
    
    Secondly, also introduce another config IS_EPOCH_S_TRULY_UTC, which
    means that the logged time in epoch_s format is actually in UTC.
    Currently (as the default) is that it it is in the local (superset
    webserver) timezone.
    
    ** There is no backward incompatibility with thes config features since
    the default behavior hasn't changed. **
---
 superset/config.py                 | 10 ++++++++++
 superset/connectors/sqla/models.py | 16 +++++++++++-----
 superset/db_engine_specs.py        |  2 +-
 superset/utils/core.py             | 25 +++++++++++++------------
 superset/viz.py                    | 15 +++++++++------
 5 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/superset/config.py b/superset/config.py
index 93720c3..8480ef8 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -566,6 +566,16 @@ WEBDRIVER_BASEURL = 'http://0.0.0.0:8080/'
 # Send user to a link where they can report bugs
 BUG_REPORT_URL = None
 
+# What is the Last N days relative in the time selector to:
+# 'today' means it is midnight (00:00:00) of today in the local timezone
+# 'now' means it is relative to the query issue time
+DEFAULT_RELATIVE_END_TIME = 'today'
+
+# Is epoch_s/epoch_ms datetime format supposed to be considered since UTC ?
+# If not, it is sassumed then the epoch_s/epoch_ms is seconds since 1/1/1970
+# localtime (in the tz where the superset webserver is running)
+IS_EPOCH_S_TRULY_UTC = False
+
 
 try:
     if CONFIG_PATH_ENV_VAR in os.environ:
diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py
index cfbea85..41b5dc7 100644
--- a/superset/connectors/sqla/models.py
+++ b/superset/connectors/sqla/models.py
@@ -127,12 +127,13 @@ class TableColumn(Model, BaseColumn):
         return self.table
 
     def get_time_filter(self, start_dttm, end_dttm):
+        is_epoch_in_utc = config.get('IS_EPOCH_S_TRULY_UTC', False)
         col = self.get_sqla_col(label='__time')
         l = []  # noqa: E741
         if start_dttm:
-            l.append(col >= text(self.dttm_sql_literal(start_dttm)))
+            l.append(col >= text(self.dttm_sql_literal(start_dttm, is_epoch_in_utc)))
         if end_dttm:
-            l.append(col <= text(self.dttm_sql_literal(end_dttm)))
+            l.append(col <= text(self.dttm_sql_literal(end_dttm, is_epoch_in_utc)))
         return and_(*l)
 
     def get_timestamp_expression(self, time_grain):
@@ -166,7 +167,7 @@ class TableColumn(Model, BaseColumn):
                 TableColumn.column_name == lookup_column.column_name).first()
         return import_datasource.import_simple_obj(db.session, i_column, lookup_obj)
 
-    def dttm_sql_literal(self, dttm):
+    def dttm_sql_literal(self, dttm, is_epoch_in_utc):
         """Convert datetime object to a SQL expression string
 
         If database_expression is empty, the internal dttm
@@ -179,10 +180,15 @@ class TableColumn(Model, BaseColumn):
         if self.database_expression:
             return self.database_expression.format(dttm.strftime('%Y-%m-%d %H:%M:%S'))
         elif tf:
+            if is_epoch_in_utc:
+                seconds_since_epoch = dttm.timestamp()
+            else:
+                seconds_since_epoch = (dttm - datetime(1970, 1, 1)).total_seconds()
+            seconds_since_epoch = int(seconds_since_epoch)
             if tf == 'epoch_s':
-                return str((dttm - datetime(1970, 1, 1)).total_seconds())
+                return str(seconds_since_epoch)
             elif tf == 'epoch_ms':
-                return str((dttm - datetime(1970, 1, 1)).total_seconds() * 1000.0)
+                return str(seconds_since_epoch * 1000)
             return "'{}'".format(dttm.strftime(tf))
         else:
             s = self.table.database.db_engine_spec.convert_dttm(
diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py
index 937537b..3ae433c 100644
--- a/superset/db_engine_specs.py
+++ b/superset/db_engine_specs.py
@@ -138,7 +138,7 @@ class BaseEngineSpec(object):
 
     @classmethod
     def epoch_ms_to_dttm(cls):
-        return cls.epoch_to_dttm().replace('{col}', '({col}/1000.000)')
+        return cls.epoch_to_dttm().replace('{col}', '({col}/1000)')
 
     @classmethod
     def get_datatype(cls, type_code):
diff --git a/superset/utils/core.py b/superset/utils/core.py
index 3e38ea5..efeff7e 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -920,7 +920,8 @@ def ensure_path_exists(path):
 def get_since_until(time_range: Optional[str] = None,
                     since: Optional[str] = None,
                     until: Optional[str] = None,
-                    time_shift: Optional[str] = None) -> (datetime, datetime):
+                    time_shift: Optional[str] = None,
+                    relative_end: Optional[str] = None) -> (datetime, datetime):
     """Return `since` and `until` date time tuple from string representations of
     time_range, since, until and time_shift.
 
@@ -946,13 +947,13 @@ def get_since_until(time_range: Optional[str] = None,
 
     """
     separator = ' : '
-    today = parse_human_datetime('today')
+    relative_end = parse_human_datetime(relative_end if relative_end else 'today')
     common_time_frames = {
-        'Last day': (today - relativedelta(days=1), today),
-        'Last week': (today - relativedelta(weeks=1), today),
-        'Last month': (today - relativedelta(months=1), today),
-        'Last quarter': (today - relativedelta(months=3), today),
-        'Last year': (today - relativedelta(years=1), today),
+        'Last day': (relative_end - relativedelta(days=1), relative_end),
+        'Last week': (relative_end - relativedelta(weeks=1), relative_end),
+        'Last month': (relative_end - relativedelta(months=1), relative_end),
+        'Last quarter': (relative_end - relativedelta(months=3), relative_end),
+        'Last year': (relative_end - relativedelta(years=1), relative_end),
     }
 
     if time_range:
@@ -969,17 +970,17 @@ def get_since_until(time_range: Optional[str] = None,
         else:
             rel, num, grain = time_range.split()
             if rel == 'Last':
-                since = today - relativedelta(**{grain: int(num)})
-                until = today
+                since = relative_end - relativedelta(**{grain: int(num)})
+                until = relative_end
             else:  # rel == 'Next'
-                since = today
-                until = today + relativedelta(**{grain: int(num)})
+                since = relative_end
+                until = relative_end + relativedelta(**{grain: int(num)})
     else:
         since = since or ''
         if since:
             since = add_ago_to_since(since)
         since = parse_human_datetime(since)
-        until = parse_human_datetime(until or 'now')
+        until = parse_human_datetime(until) if until else relative_end
 
     if time_shift:
         time_shift = parse_human_timedelta(time_shift)
diff --git a/superset/viz.py b/superset/viz.py
index dc84630..51fab12 100644
--- a/superset/viz.py
+++ b/superset/viz.py
@@ -59,6 +59,7 @@ from superset.utils.core import (
 
 config = app.config
 stats_logger = config.get('STATS_LOGGER')
+relative_end = config.get('DEFAULT_RELATIVE_END_TIME', 'today')
 
 METRIC_KEYS = [
     'metric', 'metrics', 'percent_metrics', 'metric_2', 'secondary_metric',
@@ -280,9 +281,10 @@ class BaseViz(object):
         # default order direction
         order_desc = form_data.get('order_desc', True)
 
-        since, until = utils.get_since_until(form_data.get('time_range'),
-                                             form_data.get('since'),
-                                             form_data.get('until'))
+        since, until = utils.get_since_until(relative_end=relative_end,
+                                             time_range=form_data.get('time_range'),
+                                             since=form_data.get('since'),
+                                             until=form_data.get('until'))
         time_shift = form_data.get('time_shift', '')
         self.time_shift = utils.parse_human_timedelta(time_shift)
         from_dttm = None if since is None else (since - self.time_shift)
@@ -795,9 +797,10 @@ class CalHeatmapViz(BaseViz):
                 values[str(v / 10**9)] = obj.get(metric)
             data[metric] = values
 
-        start, end = utils.get_since_until(form_data.get('time_range'),
-                                           form_data.get('since'),
-                                           form_data.get('until'))
+        start, end = utils.get_since_until(relative_end=relative_end,
+                                           time_range=form_data.get('time_range'),
+                                           since=form_data.get('since'),
+                                           until=form_data.get('until'))
         if not start or not end:
             raise Exception('Please provide both time bounds (Since and Until)')
         domain = form_data.get('domain_granularity')


Mime
View raw message