trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dbirds...@apache.org
Subject [02/14] incubator-trafodion git commit: add a new regression test for upstats for native hbase tables
Date Mon, 21 Sep 2015 23:12:17 GMT
add a new regression test for upstats for native hbase tables


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/890c306d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/890c306d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/890c306d

Branch: refs/heads/master
Commit: 890c306d2aa99635581d4abe5215ccd08fade280
Parents: 3d212c6
Author: Qifan Chen <qfc@dev02.trafodion.org>
Authored: Thu Sep 17 14:49:24 2015 +0000
Committer: Qifan Chen <qfc@dev02.trafodion.org>
Committed: Thu Sep 17 14:49:24 2015 +0000

----------------------------------------------------------------------
 core/sql/regress/seabase/EXPECTED028            | 111 +++++++++++++++++++
 core/sql/regress/seabase/FILTER028              |  96 ++++++++++++++++
 core/sql/regress/seabase/TEST028                |  44 ++++++++
 core/sql/regress/seabase/create_hbase_tables.sh |  29 +++++
 core/sql/regress/seabase/drop_hbase_tables.sh   |   7 ++
 core/sql/regress/tools/runregr_seabase.ksh      |   5 +
 core/sql/ustat/hs_la.cpp                        |   9 +-
 core/sql/ustat/hs_parser.cpp                    |  10 +-
 core/sql/ustat/hs_read.cpp                      |   3 +-
 9 files changed, 306 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/regress/seabase/EXPECTED028
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/EXPECTED028 b/core/sql/regress/seabase/EXPECTED028
new file mode 100644
index 0000000..f49c05a
--- /dev/null
+++ b/core/sql/regress/seabase/EXPECTED028
@@ -0,0 +1,111 @@
+>>
+>>log off;
+>>
+>>update statistics for table hbase."_CELL_"."test028tbl2" on every column;
+
+--- SQL operation complete.
+>>update statistics for table hbase."_CELL_"."test028tbl2" on every key;
+
+*** WARNING[9202] UPDATE STATISTICS has located previously generated histograms that are
not being regenerated. This may affect the plans that will be generated. Missing columns lists
are (COL_VALUE),(COL_TIMESTAMP),(COL_NAME),(COL_FAMILY).
+
+--- SQL operation completed with warnings.
+>>
+>>showstats for table hbase."_CELL_"."test028tbl2" on ROW_ID detail;
+
+Detailed Histogram data for Table HBASE."_CELL_"."test028tbl2"
+Table ID: 961595648449090953
+
+Hist ID:    1150301280
+Column(s):  ROW_ID
+Total Rows: 10
+Total UEC:  10
+Low Value:  ('100')
+High Value: ('9990')
+Intervals:  10
+
+Number    Rowcount         UEC Boundary
+====== =========== =========== ======================================
+     0           0           0 ('100')
+     1           1           1 ('100')
+     2           1           1 ('200')
+     3           1           1 ('300')
+     4           1           1 ('400')
+     5           1           1 ('500')
+     6           1           1 ('600')
+     7           1           1 ('700')
+     8           1           1 ('720')
+     9           1           1 ('999')
+    10           1           1 ('9990')
+
+
+--- SQL operation complete.
+>>showstats for table hbase."_CELL_"."test028tbl2" on COL_FAMILY detail;
+
+Detailed Histogram data for Table HBASE."_CELL_"."test028tbl2"
+Table ID: 961595648449090953
+
+Hist ID:    1150301276
+Column(s):  COL_FAMILY
+Total Rows: 10
+Total UEC:  1
+Low Value:  ('cf1')
+High Value: ('cf1')
+Intervals:  1
+
+Number    Rowcount         UEC Boundary
+====== =========== =========== ======================================
+     0           0           0 ('cf1')
+     1          10           1 ('cf1')
+
+
+--- SQL operation complete.
+>>showstats for table hbase."_CELL_"."test028tbl2" on COL_VALUE detail;
+
+Detailed Histogram data for Table HBASE."_CELL_"."test028tbl2"
+Table ID: 961595648449090953
+
+Hist ID:    1150301261
+Column(s):  COL_VALUE
+Total Rows: 10
+Total UEC:  10
+Low Value:  ('v100')
+High Value: ('v9990')
+Intervals:  10
+
+Number    Rowcount         UEC Boundary
+====== =========== =========== ======================================
+     0           0           0 ('v100')
+     1           1           1 ('v100')
+     2           1           1 ('v200')
+     3           1           1 ('v300')
+     4           1           1 ('v400')
+     5           1           1 ('v500')
+     6           1           1 ('v600')
+     7           1           1 ('v700')
+     8           1           1 ('v720')
+     9           1           1 ('v999')
+    10           1           1 ('v9990')
+
+
+--- SQL operation complete.
+>>
+>>explain options 'f' select * from hbase."_CELL_"."test028tbl2";
+
+LC   RC   OP   OPERATOR              OPT       DESCRIPTION           CARD
+---- ---- ---- --------------------  --------  --------------------  ---------
+
+1    .    2    root                                                  1.00E+001
+.    .    1    hbase_scan                      test028tbl2           1.00E+001
+
+--- SQL operation complete.
+>>explain options 'f' select * from hbase."_CELL_"."test028tbl2" where ROW_ID = '700';
+
+LC   RC   OP   OPERATOR              OPT       DESCRIPTION           CARD
+---- ---- ---- --------------------  --------  --------------------  ---------
+
+1    .    2    root                  o                               1.00E+000
+.    .    1    hbase_scan                      test028tbl2           1.00E+000
+
+--- SQL operation complete.
+>>
+>>log off;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/regress/seabase/FILTER028
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/FILTER028 b/core/sql/regress/seabase/FILTER028
new file mode 100755
index 0000000..590eeb9
--- /dev/null
+++ b/core/sql/regress/seabase/FILTER028
@@ -0,0 +1,96 @@
+#! /bin/sh
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+# Like the pattern-masking of QACOMP, here we filter out such stuff as
+# timestamps and generated identifiers.  Called by the runregr script
+# before doing diff.
+
+fil=$1
+if [ "$fil" = "" ]; then
+  echo "Usage: $0 filename"
+  exit 1
+fi
+
+# Remove following types of lines:
+#   Lines started with "Start Time", "End Time", "Elapsed Time",
+#                      "Execution Time", "Table ID: ddddddddd", and
+#                      "Hist ID: ddddddddd"
+ON_A_VM=
+SYSVENDOR="$(cat /sys/class/dmi/id/sys_vendor)"
+if [[ ( "${SYSVENDOR/OpenStack/}" != "$SYSVENDOR" ) ||
+      ( "${SYSVENDOR/VMware/}"    != "$SYSVENDOR" ) ]]; then
+  # Running on an OpenStack or VMware virtual machine
+  ON_A_VM=yes
+fi
+if [[ "$SQ_BUILD_TYPE" = "release" ]]; then
+sed "
+/Start Time.*/d
+" $fil |
+sed "/End Time.*/d" |
+sed "/Elapsed Time.*/d" |
+sed "/Execution Time.*/d" |
+sed "/Table ID.*/d" |
+sed "/Hist ID.*/d" |
+awk '
+{
+# we are looking for a less than 1 ms compile time
+#   but we accept time up to 1.4 ms because of other loads
+#   and time up to 2.0 ms on a virtual machine
+#   when the test is being run
+# Acceptable compile time is no more than 00:00:01.001399
+#   or on a virtual machine, no more than 00:00:01.001999
+# Examples of patterns to classify:
+#  1.  "Compile Time 00:00:01.001047" --> FAILED (at least one second)
+#  2.  "Compile Time 00:00:00.001470" --> FAILED (more than 1ms)
+#  3.  "Compile Time 00:00:00.000870" --> SUCCESS(less than 1ms)
+#  4.  "Compile Time 00:00:00.001200" --> SUCCESS(less than 1ms)
+#  5.  "Compile Time 00:00:00.001821" --> SUCCESS(less than 1ms)  only on a VM
+   if ( $1 == "Compile"  && $2 == "Time" ) { \
+     split($3, a, "."); \
+     if ( !match(a[1], "00:00:00") ) \
+       print "FAIL (at least one second)", $0; \
+     else { \
+              split(a[2], b, ""); \
+              if ((b[1] == 0) && (b[2]==0) &&  \
+                  ((b[3] == 0) || ((b[3] == 1) && ((b[4] < 4) || (ENVIRON["ON_A_VM"]
= "yes"))) )) \
+                  print "SUCCESS (less than 1ms)"; \
+              else \
+                  print "FAIL (at least 1ms)", $0; \
+     }; \
+   } else \
+     print; \
+}
+'
+fi
+
+if [[ "$SQ_BUILD_TYPE" = "debug" ]]; then
+sed "
+/Start Time.*/d
+" $fil |
+sed "/End Time.*/d" |
+sed "/Elapsed Time.*/d" |
+sed "/Execution Time.*/d" |
+sed "/Table ID.*/d" |
+sed "/Hist ID.*/d" |
+sed "/Compile Time.*/d"
+fi
+

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/regress/seabase/TEST028
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/TEST028 b/core/sql/regress/seabase/TEST028
new file mode 100644
index 0000000..64cbd38
--- /dev/null
+++ b/core/sql/regress/seabase/TEST028
@@ -0,0 +1,44 @@
+-- tests for updating statistics for external native hbase tables.
+--
+-- @@@ START COPYRIGHT @@@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+-- @@@ END COPYRIGHT @@@
+
+log LOG028 clear;
+
+log off;
+sh sh drop_hbase_tables.sh;
+sh sh create_hbase_tables.sh;
+
+log LOG028;
+
+update statistics for table hbase."_CELL_"."test028tbl2" on every column;
+update statistics for table hbase."_CELL_"."test028tbl2" on every key;
+
+showstats for table hbase."_CELL_"."test028tbl2" on ROW_ID detail;
+showstats for table hbase."_CELL_"."test028tbl2" on COL_FAMILY detail;
+showstats for table hbase."_CELL_"."test028tbl2" on COL_VALUE detail;
+
+explain options 'f' select * from hbase."_CELL_"."test028tbl2";
+explain options 'f' select * from hbase."_CELL_"."test028tbl2" where ROW_ID = '700';
+
+log off;
+sh sh drop_hbase_tables.sh;
+

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/regress/seabase/create_hbase_tables.sh
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/create_hbase_tables.sh b/core/sql/regress/seabase/create_hbase_tables.sh
new file mode 100644
index 0000000..cc44104
--- /dev/null
+++ b/core/sql/regress/seabase/create_hbase_tables.sh
@@ -0,0 +1,29 @@
+
+swhbase << EOF
+
+create 'test028tbl2', {NAME => 'cf1', VERSIONS => 1 }, {SPLITS => ['333','666','999']}
+
+tbl2=get_table 'test028tbl2'
+
+# region 0
+tbl2.put '100', 'cf1:#1', 'v100'
+tbl2.put '200', 'cf1:#1', 'v200'
+tbl2.put '300', 'cf1:#1', 'v300'
+
+# region 1
+tbl2.put '400', 'cf1:#1', 'v400'
+tbl2.put '500', 'cf1:#1', 'v500'
+tbl2.put '600', 'cf1:#1', 'v600'
+
+# region 2
+tbl2.put '700', 'cf1:#1', 'v700'
+tbl2.put '720', 'cf1:#1', 'v720'
+
+# region 3
+tbl2.put '999', 'cf1:#1', 'v999'
+tbl2.put '9990', 'cf1:#1', 'v9990'
+
+tbl2.scan
+
+
+EOF

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/regress/seabase/drop_hbase_tables.sh
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/drop_hbase_tables.sh b/core/sql/regress/seabase/drop_hbase_tables.sh
new file mode 100644
index 0000000..f9853e7
--- /dev/null
+++ b/core/sql/regress/seabase/drop_hbase_tables.sh
@@ -0,0 +1,7 @@
+
+swhbase << EOF
+
+disable 'test028tbl2'
+drop 'test028tbl2'
+
+EOF

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/regress/tools/runregr_seabase.ksh
----------------------------------------------------------------------
diff --git a/core/sql/regress/tools/runregr_seabase.ksh b/core/sql/regress/tools/runregr_seabase.ksh
index 184412f..c3f2675 100755
--- a/core/sql/regress/tools/runregr_seabase.ksh
+++ b/core/sql/regress/tools/runregr_seabase.ksh
@@ -209,6 +209,11 @@ else
   done
 fi
 
+echo "copying scripts to $REGRRUNDIR"
+cp $REGRTSTDIR/create_hbase_tables.sh $REGRRUNDIR 2>$NULL
+cp $REGRTSTDIR/drop_hbase_tables.sh $REGRRUNDIR 2>$NULL
+cp $REGRTSTDIR/FILTER028 $REGRRUNDIR 2>$NULL
+
 skippedfiles=
 
 ##############################################################

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/ustat/hs_la.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_la.cpp b/core/sql/ustat/hs_la.cpp
index 77a5ed1..a4c9e7a 100644
--- a/core/sql/ustat/hs_la.cpp
+++ b/core/sql/ustat/hs_la.cpp
@@ -1002,7 +1002,14 @@ Lng32 HSHiveTableDef::DescribeColumnNames()
 //
 NAString HSHbaseTableDef::getHistLoc(formatType format) const
 {
-  return HBASE_STATS_CATALOG "." HBASE_STATS_SCHEMA;
+  if ( HSGlobalsClass::isNativeHbaseCat(getCatName(format))) {
+    return HBASE_STATS_CATALOG "." HBASE_STATS_SCHEMA;
+  } else {  
+    NAString name(getCatName(format));
+    name.append(".");
+    name.append(getSchemaName(format));
+    return name;
+  }
 }
 
 static 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/ustat/hs_parser.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_parser.cpp b/core/sql/ustat/hs_parser.cpp
index 82158dc..00dfe8f 100644
--- a/core/sql/ustat/hs_parser.cpp
+++ b/core/sql/ustat/hs_parser.cpp
@@ -442,15 +442,13 @@ Lng32 AddTableName( const hs_table_type type
         NABoolean isHbaseOrHive = HSGlobalsClass::isHbaseCat(catName) ||
                                   HSGlobalsClass::isHiveCat(catName);
 
-        if (isHbaseOrHive)
+        if (isHbaseOrHive) {
           hs_globals->hstogram_table->append(".").append(HBASE_HIST_NAME);
-        else
-          hs_globals->hstogram_table->append(".HISTOGRAMS");
-
-        if (isHbaseOrHive)
           hs_globals->hsintval_table->append(".").append(HBASE_HISTINT_NAME);
-        else
+        } else {
+          hs_globals->hstogram_table->append(".HISTOGRAMS");
           hs_globals->hsintval_table->append(".HISTOGRAM_INTERVALS");
+        }
       }
     else
       {

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/890c306d/core/sql/ustat/hs_read.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_read.cpp b/core/sql/ustat/hs_read.cpp
index 4970d64..9f72d8c 100644
--- a/core/sql/ustat/hs_read.cpp
+++ b/core/sql/ustat/hs_read.cpp
@@ -921,7 +921,8 @@ Lng32 FetchHistograms( const QualifiedName & qualifiedName
                 getHistogramsTableLocation
                 (tabDef->getHistLoc(HSTableDef::EXTERNAL_FORMAT),
                  tabDef->isInMemoryObjectDefn());
-              if (HSGlobalsClass::isHbaseCat(objectName->getCatalogNamePart().getInternalName()))
+              if (HSGlobalsClass::isHbaseCat(objectName->getCatalogNamePart().getInternalName())
||
+                  HSGlobalsClass::isHiveCat(objectName->getCatalogNamePart().getInternalName()))
                 {
                   histogramTableName = histLoc + "." + HBASE_HIST_NAME;
                   histintsTableName  = histLoc + "." + HBASE_HISTINT_NAME;


Mime
View raw message