trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ansha...@apache.org
Subject [1/9] incubator-trafodion git commit: JIRA TRAFODION-2013 Hive null values are now handled correctly.
Date Sat, 28 May 2016 01:12:56 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master 1fa468760 -> d19936293


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/regress/hive/TEST018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018
index c0767b3..034a2fb 100644
--- a/core/sql/regress/hive/TEST018
+++ b/core/sql/regress/hive/TEST018
@@ -83,6 +83,8 @@ drop table customer_address_NOPK cascade;
 drop table customer_salt;
 drop table store_sales_salt;
 drop table nulls;
+drop table null_format_src;
+
 sh regrhadoop.ksh fs -rm   /bulkload/merged_customer_address.gz ;
 sh regrhadoop.ksh fs -rm   /bulkload/merged_customer_demogs.gz ;
 sh regrhadoop.ksh fs -rm   /bulkload/merged_customer_demogs_3;
@@ -92,6 +94,7 @@ sh  regrhbase.ksh $REGRTSTDIR/TEST018_drop_hbase_objects.hbase &>
$REGRRUNDIR/TE
 ?section setup
 --------------------------------------------------------------------------
 create table nulls (a char(5), b char(10));
+create table null_format_src (a varchar(5), b varchar(5));
 
 create table customer_demographics
 (
@@ -206,6 +209,11 @@ salt using 4 partitions
 insert into nulls values ('aaa1','bbbbb1'),(null,'bbbbb2'),
                          ('aaa3',null),(null,null),
                           ('aaa5',null),(null,'bbbbb6');
+
+insert into null_format_src values 
+   ('a', 'b'), ('a', null), (null, 'b'), ('a', ''), ('', 'b'),
+   (null, ''), (':', null), ('', ''), (':', ':'), (null, null);
+
 --load1
 load with no recovery into customer_address 
 select * from hive.hive.customer_address;    
@@ -237,22 +245,42 @@ select * from hive.hive.store_sales where ss_item_sk <= 1000;
 --
 --select count(*) from hive.hive.store_sales  where ss_item_sk <= 1000;
 select count(*) from store_sales_salt;
--------------------------------
---hive insert 0
-insert overwrite table hive.hive.nulls select * from nulls;
-select * from hive.hive.nulls order by a,b;
+
+------------------------------------------
+-- handling of various null formats
+
+-- using insert
+insert overwrite table hive.hive.null_format_default select * from null_format_src;
+select * from hive.hive.null_format_default;
+
+insert overwrite table hive.hive.null_format_empty select * from null_format_src;
+select * from hive.hive.null_format_empty;
+
+insert overwrite table hive.hive.null_format_colon select * from null_format_src;
+select * from hive.hive.null_format_colon;
+
+
+-- using unload
+unload with purgedata from target
+   into '/user/hive/exttables/null_format_default'
+   select * from null_format_src;
+select * from hive.hive.null_format_default;
+
+unload with purgedata from target
+   into '/user/hive/exttables/null_format_empty'
+   select * from null_format_src;
+select * from hive.hive.null_format_empty;
+
+unload with purgedata from target
+   into '/user/hive/exttables/null_format_colon'
+   select * from null_format_src;
+select * from hive.hive.null_format_colon;
 
 ---index 0
 create index customer_idx1 on customer_salt(c_first_name, c_last_name);
 
-
 sh  regrhbase.ksh $REGRTSTDIR/TEST018_create_hbase_objects.hbase &> $REGRRUNDIR/LOG018_create_hbase_tables.log
;
 
----unload 0 -- nulls 
-unload with purgedata from target
-   into '/user/hive/exttables/nulls'
-   select * from nulls;
-select * from hive.hive.nulls order by a,b;
 
 --exp1
 explain options 'f' 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/regress/hive/TEST018_create_hive_tables.hive
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST018_create_hive_tables.hive b/core/sql/regress/hive/TEST018_create_hive_tables.hive
index 0b1ee22..8536030 100644
--- a/core/sql/regress/hive/TEST018_create_hive_tables.hive
+++ b/core/sql/regress/hive/TEST018_create_hive_tables.hive
@@ -148,14 +148,18 @@ create external table unload_store_sales_summary
 row format delimited fields terminated by '|' 
 location  '/user/hive/exttables/unload_store_sales_summary';
 
-drop table nulls;
-create external table nulls
-(    a string, 
-     b string
-) 
+drop table null_format_default;
+create external table null_format_default (a string, b string);
+
+drop table null_format_empty;
+create external table null_format_empty(a string, b string)
 row format delimited fields terminated by '|' 
-location  '/user/hive/exttables/nulls';
+null defined as '';
 
+drop table null_format_colon;
+create external table null_format_colon(a string, b string)
+row format delimited fields terminated by '|' 
+null defined as ':';
 
 quit;
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/regress/tools/runregr_hive.ksh
----------------------------------------------------------------------
diff --git a/core/sql/regress/tools/runregr_hive.ksh b/core/sql/regress/tools/runregr_hive.ksh
index 1165892..f6fd5f9 100755
--- a/core/sql/regress/tools/runregr_hive.ksh
+++ b/core/sql/regress/tools/runregr_hive.ksh
@@ -225,43 +225,8 @@ skippedfiles=
 testfiles="$prettyfiles"
 prettyfiles=
 
-# skip these tests on all platforms.
-skipTheseTests=""
-
-# skip these tests for SQLMX and SQLMP tables on NSK platform
-if [ $nsk -eq 1 ]; then
-  skipTheseTests="TEST*"
-fi
-
-# skip these tests for RELEASE ONLY tests on NSK
-#if [ "$BUILD_FLAVOR" = "RELEASE" ]; then
-#  skipTheseTests="$skipTheseTests TESTxxx"
-#fi
-
-# skip these tests for DEBUG ONLY tests on NSK
-#if [ "$BUILD_FLAVOR" = "DEBUG" ]; then
-#  skipTheseTests="$skipTheseTests TESTxxx"
-#fi
-
-# skip these tests for NT
-if [ `uname` = "Windows_NT" ]; then
-  skipTheseTests="TEST*"
-fi
-
-
-#skip these tests for Linux
-#if [ `uname` = "Linux" ]; then
-#  skipTheseTests="$skipTheseTests TESTxxx"
-#fi
-
-#skip these tests when seabase eq 1.
-if [ "$seabase" -ne 0 ]; then
-  if [ "$seabase" -eq 1 ]; then
-    skipTheseTests="TEST004"
-  fi
-fi
-#skipTheseTests="TEST020"
-skipTheseTests=
+# skip these tests
+skipTheseTests="TEST020"
 
 # Skip exclusive tests during concurrent execution
 if [ "$REGRCONCURRENT" -ne 0 ]; then

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index 8faee51..5878bf7 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -3646,7 +3646,6 @@ enum DefaultConstants
   GROUP_BY_PARTIAL_ROOT_THRESHOLD,
   TRAF_UNLOAD_BYPASS_LIBHDFS,
   TRAF_UNLOAD_DEF_DELIMITER,
-  TRAF_UNLOAD_DEF_NULL_STRING,
   TRAF_UNLOAD_DEF_RECORD_SEPARATOR,
   TRAF_LOAD_FORCE_CIF,
   TRAF_ENABLE_ORC_FORMAT,
@@ -3818,6 +3817,7 @@ enum DefaultConstants
   HIVE_SCAN_SPECIAL_MODE,
 
   COMPILER_IDLE_TIMEOUT,
+
   // This enum constant must be the LAST one in the list; it's a count,
   // not an Attribute (it's not IN DefaultDefaults; it's the SIZE of it)!
   __NUM_DEFAULT_ATTRIBUTES

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a549e0e/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 2d2b83b..6221024 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -3404,7 +3404,6 @@ XDDkwd__(SUBQUERY_UNNESTING,			"ON"),
 
   DDkwd__(TRAF_UNLOAD_BYPASS_LIBHDFS,                  "ON"),
   DD_____(TRAF_UNLOAD_DEF_DELIMITER,                   "|" ),
-  DD_____(TRAF_UNLOAD_DEF_NULL_STRING,                 "" ),
   DD_____(TRAF_UNLOAD_DEF_RECORD_SEPARATOR,            "\n" ),
   DDint__(TRAF_UNLOAD_HDFS_COMPRESS,                   "0"),
   DDkwd__(TRAF_UNLOAD_SKIP_WRITING_TO_FILES,           "OFF"),


Mime
View raw message