impala-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tarmstr...@apache.org
Subject [2/3] incubator-impala git commit: IMPALA-3504: UDF for current timestamp in UTC
Date Fri, 07 Jul 2017 05:02:02 GMT
IMPALA-3504: UDF for current timestamp in UTC

This change adds a UDF "utc_timestamp" which returns the current
date and time in UTC. Example query:

select utc_timestamp();

+-------------------------------+
| utc_timestamp()               |
+-------------------------------+
| 2017-06-15 17:36:39.290773000 |
+-------------------------------+

Change-Id: I969fc805922f2bb9c8101e84f85ff2cc3b1b6729
Reviewed-on: http://gerrit.cloudera.org:8080/7203
Tested-by: Impala Public Jenkins
Reviewed-by: Matthew Jacobs <mj@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/9037b8e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/9037b8e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/9037b8e3

Branch: refs/heads/master
Commit: 9037b8e38598a9710cd2dfcf30b9fd353bdf6191
Parents: 4e17839
Author: Bikramjeet Vig <bikramjeet.vig@cloudera.com>
Authored: Thu Jun 15 10:29:02 2017 -0700
Committer: Matthew Jacobs <mj@cloudera.com>
Committed: Thu Jul 6 23:04:28 2017 +0000

----------------------------------------------------------------------
 be/src/exprs/expr-test.cc                       | 31 ++++++++++++++++++++
 be/src/exprs/timestamp-functions-ir.cc          |  7 +++++
 be/src/exprs/timestamp-functions.h              |  1 +
 be/src/runtime/runtime-state.cc                 |  3 ++
 be/src/runtime/runtime-state.h                  |  8 +++--
 be/src/runtime/timestamp-value.h                |  8 +++++
 be/src/service/impala-server.cc                 |  6 +++-
 common/function-registry/impala_functions.py    |  1 +
 common/thrift/ImpalaInternalService.thrift      |  6 +++-
 .../org/apache/impala/testutil/TestUtils.java   |  7 ++++-
 10 files changed, 73 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index a8998ec..a1fd9a1 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -3864,6 +3864,7 @@ TEST_F(ExprTest, UtilityFunctions) {
   TestStringValue("typeOf(cast(10 as DOUBLE))", "DOUBLE");
   TestStringValue("typeOf(current_database())", "STRING");
   TestStringValue("typeOf(now())", "TIMESTAMP");
+  TestStringValue("typeOf(utc_timestamp())", "TIMESTAMP");
   TestStringValue("typeOf(cast(10 as DECIMAL))", "DECIMAL(9,0)");
   TestStringValue("typeOf(0.0)", "DECIMAL(1,1)");
   TestStringValue("typeOf(3.14)", "DECIMAL(3,2)");
@@ -5229,6 +5230,7 @@ TEST_F(ExprTest, TimestampFunctions) {
 
   // Test functions with unknown expected value.
   TestValidTimestampValue("now()");
+  TestValidTimestampValue("utc_timestamp()");
   TestValidTimestampValue("current_timestamp()");
   TestValidTimestampValue("cast(unix_timestamp() as timestamp)");
 
@@ -5264,6 +5266,10 @@ TEST_F(ExprTest, TimestampFunctions) {
   timestamp_result = ConvertValue<TimestampValue>(GetValue("current_timestamp()",
       TYPE_TIMESTAMP));
   EXPECT_BETWEEN(start_time, timestamp_result, TimestampValue::LocalTime());
+  const TimestampValue utc_start_time = TimestampValue::UtcTime();
+  timestamp_result = ConvertValue<TimestampValue>(GetValue("utc_timestamp()",
+      TYPE_TIMESTAMP));
+  EXPECT_BETWEEN(utc_start_time, timestamp_result, TimestampValue::UtcTime());
   // UNIX_TIMESTAMP() has second precision so the comparison start time is shifted back
   // a second to ensure an earlier value.
   unix_start_time =
@@ -5273,6 +5279,31 @@ TEST_F(ExprTest, TimestampFunctions) {
   EXPECT_BETWEEN(TimestampValue::FromUnixTime(unix_start_time - 1), timestamp_result,
       TimestampValue::LocalTime());
 
+  // Test that UTC and local time represent the same point in time
+  {
+    const string stmt = "select now(), utc_timestamp()";
+    vector<FieldSchema> result_types;
+    Status status = executor_->Exec(stmt, &result_types);
+    EXPECT_TRUE(status.ok()) << "stmt: " << stmt << "\nerror: " <<
status.GetDetail();
+    DCHECK(result_types.size() == 2);
+    EXPECT_EQ(TypeToOdbcString(TYPE_TIMESTAMP), result_types[0].type)
+        << "invalid type returned by now()";
+    EXPECT_EQ(TypeToOdbcString(TYPE_TIMESTAMP), result_types[1].type)
+        << "invalid type returned by utc_timestamp()";
+    string result_row;
+    status = executor_->FetchResult(&result_row);
+    EXPECT_TRUE(status.ok()) << "stmt: " << stmt << "\nerror: " <<
status.GetDetail();
+    vector<string> result_cols;
+    boost::split(result_cols, result_row, boost::is_any_of("\t"));
+    // To ensure this fails if columns are not tab separated
+    DCHECK(result_cols.size() == 2);
+    const TimestampValue local_time = ConvertValue<TimestampValue>(result_cols[0]);
+    const TimestampValue utc_timestamp = ConvertValue<TimestampValue>(result_cols[1]);
+    TimestampValue utc_converted_to_local(utc_timestamp);
+    utc_converted_to_local.UtcToLocal();
+    EXPECT_EQ(utc_converted_to_local, local_time);
+  }
+
   // Test alias
   TestValue("now() = current_timestamp()", TYPE_BOOLEAN, true);
 

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/timestamp-functions-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions-ir.cc b/be/src/exprs/timestamp-functions-ir.cc
index 6a25ced..0cbf19e 100644
--- a/be/src/exprs/timestamp-functions-ir.cc
+++ b/be/src/exprs/timestamp-functions-ir.cc
@@ -304,6 +304,13 @@ TimestampVal TimestampFunctions::Now(FunctionContext* context) {
   return return_val;
 }
 
+TimestampVal TimestampFunctions::UtcTimestamp(FunctionContext* context) {
+  const TimestampValue* utc_timestamp = context->impl()->state()->utc_timestamp();
+  TimestampVal return_val;
+  utc_timestamp->ToTimestampVal(&return_val);
+  return return_val;
+}
+
 // Writes 'num' as ASCII into 'dst'. If necessary, adds leading zeros to make the ASCII
 // representation exactly 'len' characters. Both 'num' and 'len' must be >= 0.
 static inline void IntToChar(uint8_t* dst, int num, int len) {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/exprs/timestamp-functions.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timestamp-functions.h b/be/src/exprs/timestamp-functions.h
index cbd8b98..9e447b8 100644
--- a/be/src/exprs/timestamp-functions.h
+++ b/be/src/exprs/timestamp-functions.h
@@ -140,6 +140,7 @@ class TimestampFunctions {
 
   /// Date/time functions.
   static TimestampVal Now(FunctionContext* context);
+  static TimestampVal UtcTimestamp(FunctionContext* context);
   static StringVal ToDate(FunctionContext* context, const TimestampVal& ts_val);
   static IntVal DateDiff(FunctionContext* context, const TimestampVal& ts_val1,
       const TimestampVal& ts_val2);

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/runtime-state.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.cc b/be/src/runtime/runtime-state.cc
index b05fc6b..89eec29 100644
--- a/be/src/runtime/runtime-state.cc
+++ b/be/src/runtime/runtime-state.cc
@@ -78,6 +78,8 @@ RuntimeState::RuntimeState(QueryState* query_state, const TPlanFragmentCtx&
frag
     fragment_ctx_(&fragment_ctx),
     instance_ctx_(&instance_ctx),
     now_(new TimestampValue(TimestampValue::Parse(query_state->query_ctx().now_string))),
+    utc_timestamp_(new TimestampValue(TimestampValue::Parse(
+        query_state->query_ctx().utc_timestamp_string))),
     exec_env_(exec_env),
     profile_(obj_pool(), "Fragment " + PrintId(instance_ctx.fragment_instance_id)),
     instance_buffer_reservation_(nullptr),
@@ -93,6 +95,7 @@ RuntimeState::RuntimeState(
     instance_ctx_(nullptr),
     local_query_state_(query_state_),
     now_(new TimestampValue(TimestampValue::Parse(qctx.now_string))),
+    utc_timestamp_(new TimestampValue(TimestampValue::Parse(qctx.utc_timestamp_string))),
     exec_env_(exec_env),
     profile_(obj_pool(), "<unnamed>"),
     instance_buffer_reservation_(nullptr),

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/runtime-state.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/runtime-state.h b/be/src/runtime/runtime-state.h
index 186d540..9a1d0b2 100644
--- a/be/src/runtime/runtime-state.h
+++ b/be/src/runtime/runtime-state.h
@@ -115,6 +115,7 @@ class RuntimeState {
     return query_ctx().session.connected_user;
   }
   const TimestampValue* now() const { return now_.get(); }
+  const TimestampValue* utc_timestamp() const { return utc_timestamp_.get(); }
   void set_now(const TimestampValue* now);
   const TUniqueId& query_id() const { return query_ctx().query_id; }
   const TUniqueId& fragment_instance_id() const {
@@ -338,9 +339,12 @@ class RuntimeState {
   /// Provides instance id if instance_ctx_ == nullptr
   TUniqueId no_instance_id_;
 
-  /// Query-global timestamp, e.g., for implementing now(). Set from query_globals_.
-  /// Use pointer to avoid inclusion of timestampvalue.h and avoid clang issues.
+  /// Query-global timestamps for implementing now() and utc_timestamp(). Both represent
+  /// the same point in time but now_ is in local time and utc_timestamp_ is in UTC.
+  /// Set from query_globals_. Use pointer to avoid inclusion of timestampvalue.h and
+  /// avoid clang issues.
   boost::scoped_ptr<TimestampValue> now_;
+  boost::scoped_ptr<TimestampValue> utc_timestamp_;
 
   /// TODO: get rid of this and use ExecEnv::GetInstance() instead
   ExecEnv* exec_env_;

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/runtime/timestamp-value.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/timestamp-value.h b/be/src/runtime/timestamp-value.h
index 83c38e9..c84eb7f 100644
--- a/be/src/runtime/timestamp-value.h
+++ b/be/src/runtime/timestamp-value.h
@@ -123,6 +123,14 @@ class TimestampValue {
     return TimestampValue(boost::posix_time::microsec_clock::local_time());
   }
 
+  /// Returns the current Coordinated Universal Time ("UTC") with microsecond accuracy.
+  /// This should not be used to time something because it is affected by adjustments to
+  /// the system clock such as a manual correction by a system admin. For timings,
+  /// use functions in util/time.h.
+  static TimestampValue UtcTime() {
+    return TimestampValue(boost::posix_time::microsec_clock::universal_time());
+  }
+
   /// Returns a TimestampValue converted from a TimestampVal. The caller must ensure
   /// the TimestampVal does not represent a NULL.
   static TimestampValue FromTimestampVal(const impala_udf::TimestampVal& udf_value) {

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 37e06ac..cf91ba3 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -875,7 +875,11 @@ Status ImpalaServer::ExecuteInternal(
 
 void ImpalaServer::PrepareQueryContext(TQueryCtx* query_ctx) {
   query_ctx->__set_pid(getpid());
-  query_ctx->__set_now_string(TimestampValue::LocalTime().ToString());
+  TimestampValue utc_timestamp = TimestampValue::UtcTime();
+  query_ctx->__set_utc_timestamp_string(utc_timestamp.ToString());
+  TimestampValue local_timestamp(utc_timestamp);
+  local_timestamp.UtcToLocal();
+  query_ctx->__set_now_string(local_timestamp.ToString());
   query_ctx->__set_start_unix_millis(UnixMillis());
   query_ctx->__set_coord_address(MakeNetworkAddress(FLAGS_hostname, FLAGS_be_port));
 

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/common/function-registry/impala_functions.py
----------------------------------------------------------------------
diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py
index b80261c..4201553 100644
--- a/common/function-registry/impala_functions.py
+++ b/common/function-registry/impala_functions.py
@@ -227,6 +227,7 @@ visible_functions = [
       '_ZN6impala18TimestampFunctions22UnixAndFromUnixPrepareEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE',
       '_ZN6impala18TimestampFunctions20UnixAndFromUnixCloseEPN10impala_udf15FunctionContextENS2_18FunctionStateScopeE'],
   [['now', 'current_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions3NowEPN10impala_udf15FunctionContextE'],
+  [['utc_timestamp'], 'TIMESTAMP', [], '_ZN6impala18TimestampFunctions12UtcTimestampEPN10impala_udf15FunctionContextE'],
   [['from_utc_timestamp'], 'TIMESTAMP', ['TIMESTAMP', 'STRING'],
    "impala::TimestampFunctions::FromUtc"],
   [['to_utc_timestamp'], 'TIMESTAMP', ['TIMESTAMP', 'STRING'],

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/common/thrift/ImpalaInternalService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index c31cd7f..4aefe55 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -322,7 +322,7 @@ struct TQueryCtx {
   // Session state including user.
   3: required TSessionState session
 
-  // String containing a timestamp set as the query submission time.
+  // String containing a timestamp (in local timezone) set as the query submission time.
   4: required string now_string
 
   // Process ID of the impalad to which the user is connected.
@@ -371,6 +371,10 @@ struct TQueryCtx {
   // The pool to which this request has been submitted. Used to update pool statistics
   // for admission control.
   16: optional string request_pool
+
+  // String containing a timestamp (in UTC) set as the query submission time. It
+  // represents the same point in time as now_string
+  17: required string utc_timestamp_string
 }
 
 // Specification of one output destination of a plan fragment

http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9037b8e3/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index ce93984..5404455 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -22,8 +22,10 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collections;
+import java.util.Date;
 import java.util.Map;
 import java.util.Scanner;
+import java.util.TimeZone;
 
 import javax.json.Json;
 import javax.json.JsonObject;
@@ -258,7 +260,10 @@ public class TestUtils {
     queryCtx.setSession(new TSessionState(new TUniqueId(), TSessionType.BEESWAX,
         defaultDb, user, new TNetworkAddress("localhost", 0)));
     SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSSSSS");
-    queryCtx.setNow_string(formatter.format(Calendar.getInstance().getTime()));
+    Date now = Calendar.getInstance().getTime();
+    queryCtx.setNow_string(formatter.format(now));
+    formatter.setTimeZone(TimeZone.getTimeZone("GMT"));
+    queryCtx.setUtc_timestamp_string(formatter.format(now));
     queryCtx.setStart_unix_millis(System.currentTimeMillis());
     queryCtx.setPid(1000);
     // Disable rewrites by default because some analyzer tests have non-executable


Mime
View raw message