incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1163097 [5/7] - in /incubator/hcatalog/trunk: ./ src/test/e2e/ src/test/e2e/hcatalog/ src/test/e2e/hcatalog/conf/ src/test/e2e/hcatalog/data/ src/test/e2e/hcatalog/deployers/ src/test/e2e/hcatalog/drivers/ src/test/e2e/hcatalog/paramfiles/...
Date Tue, 30 Aug 2011 06:18:18 GMT
Added: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1163097&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (added)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Tue Aug 30 06:18:16 2011
@@ -0,0 +1,3006 @@
+#!/home/y/bin/perl
+
+	#
+	# Do
+	# egrep '^#|name.*=>' hcat.conf | egrep -v '^#!|egrep' | less
+	# to get an outline of this test conf file
+	#
+	
+  # Has a couple of Hive set directives:
+  #   set hive.exec.dynamic.partition.mode=nonstrict;
+  #   set hive.exec.dynamic.partition=true;
+
+
+$cfg = {
+        'driver' => 'HCat',
+
+#                'run_as' => 'hadoopqa',
+                'use-pig.pl' => 1,
+		'groups' => [
+# This first group should be moved to deployer ?
+		{
+			'name' => 'hcat_setup_Hive_createTable',
+			'tests' => [
+
+				{
+				 'num' => 1
+				,'hcat_cmdline_args' => ['-g', 'users', '-p', 'rwxrwx---'],
+				,'hcat' => "
+CREATE external TABLE IF NOT EXISTS numbers (
+INTNUM1000	int,
+ID		smallint,
+INTNUM5		tinyint,
+INTNUM100	int,
+INTNUM		int,
+LONGNUM		bigint,
+FLOATNUM	float,
+DOUBLENUM	double
+)
+stored as RCFile
+location ':INPATH:/numbers';
+"
+				,'rc'   => 0
+				}, 
+				{
+				 'num' => 6
+				,'hcat_cmdline_args' => ['-g', 'users', '-p', 'rwxrwx---'],
+				,'hcat' => "
+CREATE external TABLE IF NOT EXISTS complex (
+  mymap       map<string, string>,
+  mytuple     struct<num:int,str:string,dbl:double>,
+  bagofmap    array<map<string,string>>,
+  rownum      int
+)
+row format DELIMITED FIELDS TERMINATED BY '\001' 
+           COLLECTION ITEMS TERMINATED BY '\002'
+           MAP KEYS TERMINATED BY '\003' 
+stored as rcfile
+location ':INPATH:/complex';
+"
+				,'rc'   => 0
+				}, 
+
+				{
+				 'num' => 7
+				,'hcat_cmdline_args' => ['-g', 'users', '-p', 'rwxrwx---'],
+				,'hcat' => "
+CREATE external TABLE IF NOT EXISTS boolean_table (
+myflag boolean,
+rownum int
+)
+row format DELIMITED FIELDS TERMINATED BY '\001' 
+           COLLECTION ITEMS TERMINATED BY '\002'
+           MAP KEYS TERMINATED BY '\003' 
+stored as rcfile
+location ':INPATH:/boolean';
+"
+				,'rc'   => 0
+				}, 
+
+				{
+				 'num' => 8
+				# same as 'numbers' from above, just with 'int' for small/tiny ints.
+				,'hcat_cmdline_args' => ['-g', 'users', '-p', 'rwxrwx---'],
+				,'hcat' => "
+CREATE external TABLE IF NOT EXISTS numbers_pig (
+INTNUM1000	int,
+ID		int, -- smallint,
+INTNUM5		int, -- tinyint,
+INTNUM100	int,
+INTNUM		int,
+LONGNUM		bigint,
+FLOATNUM	float,
+DOUBLENUM	double
+)
+stored as RCFile
+location ':INPATH:/numbers_pig';
+"
+				,'rc'   => 0
+				}, 
+
+			],
+ 		}, # end g
+################################################################################
+# HIVE STORED DATA
+################################################################################
+
+#-------------------------------------------------------------------------------
+# Create partitioned test table using Hive
+#-------------------------------------------------------------------------------
+                {
+                'name' => 'hcat_hive2hive_partitioned',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'hive' => "
+CREATE TABLE tmp_hive_partitioned_:RUNID: (
+  id        smallint,
+  intnum    int,
+  floatnum  float
+)
+partitioned by (
+  idmod5    tinyint
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_hive_partitioned_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                        {
+                        'num' => 2,
+			'depends_on' => 'hcat_hive2hive_partitioned_1',
+                        'hadoop' => "fs -ls :TMP:/hcat_tmp_tables",
+                        'rc' => 0, 
+			'expected_out_regex' => ":TMP:/hcat_tmp_tables/tmp_hive_partitioned_:RUNID:",
+                        },
+
+                        {
+                        'num' => 3,
+			'depends_on' => 'hcat_hive2hive_partitioned_1',
+                        'hive' => "show tables;",
+                        'rc' => 0, 
+			'expected_out_regex' => "tmp_hive_partitioned_:RUNID:",
+                        },
+
+                        {
+                        'num' => 4,
+                        # select from empty table
+			'depends_on' => 'hcat_hive2hive_partitioned_1',
+                        'hive' => "select * from tmp_hive_partitioned_:RUNID:;",
+                        'rc' => 0, 
+			'expected_err_regex' => "OK",
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 5,
+                        # 
+			'depends_on' => 'hcat_hive2hive_partitioned_1',
+                        'hive' => "
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.exec.dynamic.partition=true;
+
+insert overwrite table tmp_hive_partitioned_:RUNID:
+partition (idmod5=1)
+select id, intnum, floatnum
+from numbers
+where id % 5 = 1;
+
+insert overwrite table tmp_hive_partitioned_:RUNID:
+partition (idmod5=2)
+select id, intnum, floatnum
+from numbers
+where id % 5 = 2;
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "OK",
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 6,
+                        # 
+			'depends_on' => 'hcat_hive2hive_partitioned_5',
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'floatpostprocess' => 1,
+                        'delimiter' => '	',
+                        'hive' => "
+select idmod5, id, intnum, floatnum 
+from tmp_hive_partitioned_:RUNID:
+;
+",
+                        'sql' => "
+select id % 5, id, intnum, floatnum
+from numbers
+where (id % 5 = 1)
+   or (id % 5 = 2)
+;
+", 
+                        },
+
+                        ] # end tests
+                }, # end group
+
+#-------------------------------------------------------------------------------
+# HIVE STORED -> HIVE SECTION
+#-------------------------------------------------------------------------------
+
+		{
+		'name' => 'hcat_hive2hive_type_check',
+		'tests' => [
+			# for how the 'typecheck' udf works.
+			# If the the typecheck does not pass, the udf will abort.
+			# Look at the jobtracker for info about the actual value, 
+			# e.g. something like: "Expected java.lang.String, got java.lang.Integer"
+			# The verification:
+			# 'expected_out_regex' => "class java",
+			# is needed because otherwise the tests might "PASS" because the output was empty...
+
+			{
+			'num' => 1,
+			'hive' => q\ 
+add jar :FUNCPATH:/testudf.jar;
+create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+select typecheck('int+smallint+tinyint+int+int+bigint+float+double',
+intnum1000,id,intnum5,intnum100,intnum,longnum,floatnum,doublenum) from numbers;
+\,
+			'rc' => 0,
+			'expected_out_regex' => "class java",
+			'expected_err_regex' => "OK",
+			'not_expected_err_regex' => "FAILED",
+			},
+
+			{
+			'num' => 2,
+			'hive' => q\ 
+add jar :FUNCPATH:/testudf.jar;
+create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+select typecheck('map<string,string>+struct<num:int,str:string,dbl:double>+array<map<string,string>>+int',
+mymap, mytuple, bagofmap, rownum) from complex;
+\,
+			'rc' => 0,
+			'expected_out_regex' => "class java",
+			'expected_err_regex' => "OK",
+			'not_expected_err_regex' => "FAILED",
+			},
+
+			{
+			'num' => 3,
+			'hive' => q\ 
+add jar :FUNCPATH:/testudf.jar;
+create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+select typecheck('boolean+int',
+myflag,rownum) from boolean_table;
+\,
+			'rc' => 0,
+			'expected_out_regex' => "class java",
+			'expected_err_regex' => "OK",
+			'not_expected_err_regex' => "FAILED",
+			},
+
+			                        {
+                        'num' => 5,
+                        'floatpostprocess' => 1,
+                        'delimiter' => '	',
+                        'hive' => q\
+select
+  id, -- expect smallint
+  intnum5 + id, -- expect smallint
+  intnum5 + id + intnum, -- expect int
+  intnum5 + id + intnum + longnum, -- expect bigint
+  intnum5 * id, -- expect smallint
+  intnum5 * id * intnum, -- expect int
+  intnum5 * id * intnum * longnum, -- expect bigint
+  intnum5 + 500, -- expect int
+  intnum5 + 1.5, -- expect float
+  cast(intnum5 + 1.5 as double), -- expect double
+  cast(intnum5 + 1.0 as int), -- expect int
+  floatnum + doublenum, -- expect double
+  floatnum * doublenum -- expect double
+from numbers
+order by id
+limit 500
+;
+\,
+                        'sql' => "
+select
+  id,
+  intnum5 + id,
+  intnum5 + id + intnum,
+  intnum5 + id + intnum + longnum,
+  intnum5 * id,
+  intnum5 * id * intnum,
+  intnum5 * id * intnum * longnum,
+  intnum5 + 500,
+  intnum5 + 1.5,
+  cast(intnum5 + 1.5 as double precision),
+  cast(intnum5 + 1.0 as integer),
+  floatnum + doublenum,
+  floatnum * doublenum
+from numbers
+order by id
+limit 500
+;
+",
+                        },
+
+                        {
+                        'num' => 6,
+                        'ignore' => '!!! Hive truncates where Postgres rounds',
+                        'hive' => q\
+select
+  id,
+  intnum5, 
+  cast(intnum5 + 1.9 as int)
+from numbers
+order by id
+limit 5
+;
+\,
+                        'sql' => "
+select
+  id,
+  intnum5, 
+  cast(intnum5 + 1.9 as integer) 
+from numbers
+order by id
+limit 5
+;
+", 
+			},
+
+        		] # end tests
+        	}, # end group
+
+		{
+		'name' => 'hcat_hive2hive_diagnostic_operators',
+		'tests' => [
+
+			{
+			'num' => 1,
+			'hive' => "show tables;",
+			'expected_out_regex' => "OK",
+			},
+			{
+			'num' => 5,
+			'hive' => "describe complex;",
+                        'sql' => "\\d", # dummy statement
+			},
+
+			{
+			'num' => 6,
+			'hive' => "describe complex.mytuple;",
+			'expected_out_regex' => 'num\s+int\s+from\s+deserializer\s*\nstr\s+string\s+from\s+deserializer\s*\ndbl\s+double\s+from\s+deserializer',
+			},
+
+			{
+			'num' => 7,
+			'hive' => "describe complex.bagofmap;",
+			'expected_out_regex' => 'bagofmap\s+array<map<string,string>>\s+from deserializer',
+			},
+
+			{
+			'num' => 8,
+			'hive' => "describe boolean_table;",
+                        'sql' => "\\d", # dummy statement
+			},
+
+			{
+			'num' => 9,
+			'hive' => "describe boolean_table.myflag;",
+			'expected_out_regex' => 'myflag\s+boolean\s+from deserializer',
+			},
+
+        		] # end tests
+        	}, # end group
+
+		{
+		'name' => 'hcat_hive2hive_select',
+                'sortBenchmark' => 1,
+                'sortResults' => 1,
+		'tests' => [
+
+                        {
+                        'num' => 1,
+                	'floatpostprocess' => 1,
+                	'delimiter' => '	',
+			# actually no nulls in table, so had not needed the casting..., 
+                        'hive' => q\
+select
+  if (intnum1000 IS NULL, "", cast(intnum1000 as STRING)),
+  if (id IS NULL, "", cast(id as STRING)),
+  if (intnum5 IS NULL, "", cast(intnum5 as STRING)),
+  if (intnum100 IS NULL, "", cast(intnum100 as STRING)),
+  if (intnum IS NULL, "", cast(intnum as STRING)),
+  if (longnum IS NULL, "", cast(longnum as STRING)),
+  if (floatnum IS NULL, "", cast(floatnum as STRING)),
+  if (doublenum IS NULL, "", cast(doublenum as STRING))
+from numbers;
+\,
+                        'sql' => q\
+select
+  intnum1000,
+  id,
+  intnum5,
+  intnum100,
+  intnum,
+  longnum,
+  floatnum,
+  doublenum
+from numbers;
+\,
+                        },
+
+                        {
+                        'num' => 2,
+                        'hive' => q\
+select
+  mymap,
+  mytuple,
+  bagofmap,
+  rownum
+from complex;
+\,
+                        'sql' => "\\d", # dummy statement
+                        },
+
+                        {
+                        'num' => 3,
+                        'hive' => q\
+select
+  myflag, rownum
+from boolean_table;
+\,
+                        'sql' => "\\d", # dummy statement
+                        },
+
+			
+			
+                        {
+                        'num' => 8,
+                	'sortBenchmark' => 1,
+	                'sortResults' => 1,
+                        'hive' => q\
+select
+  distinct 
+  intnum1000,
+  intnum5,
+  intnum100,
+  intnum,
+  longnum
+from numbers;
+\,
+                        'sql' => q\
+select
+  distinct 
+  intnum1000,
+  intnum5,
+  intnum100,
+  intnum,
+  longnum
+from numbers;
+\,
+                        },
+
+        		] # end tests
+        	}, # end group
+
+		{
+		'name' => 'hcat_hive2hive_lateral',
+                'sortBenchmark' => 1,
+                'sortResults' => 1,
+		# Had work-around for Hive problem 
+		# set hive.optimize.cp=false; 
+		'tests' => [
+
+			# NOTE: 
+			# The queries below written w/o WHERE clauses until this jira is resolved:
+			# https://issues.apache.org/jira/browse/HIVE-1056
+			# http://wiki.apache.org/hadoop/Hive/LanguageManual/LateralView
+
+                        {
+                        'num' => 1,
+                        'hive' => q\
+select rownum, item 
+from complex LATERAL VIEW explode(bagofmap) explodedTable AS item;
+\,
+                        'sql' => "\\d", # dummy statement
+			},
+
+                        
+                        {
+                        'num' => 3,
+                        'hive' => q\
+select 
+  rownum, 
+  item, 
+  "item['a1']:", if (item['a1'] IS NULL, "", cast(item['a1'] as STRING)),
+  "item['a2']:", if (item['a2'] IS NULL, "", cast(item['a2'] as STRING)),
+  "item['k1']:", if (item['k1'] IS NULL, "", cast(item['k1'] as STRING)),
+  "item['k6']:", if (item['k6'] IS NULL, "", cast(item['k6'] as STRING))
+from complex LATERAL VIEW explode(bagofmap) explodedTable AS item
+;
+\,
+                        'sql' => "\\d", # dummy statement
+			},
+
+                       
+
+        		] # end tests
+        	}, # end group
+
+		{
+		'name' => 'hcat_hive2hive_join',
+                'sortBenchmark' => 1,
+                'sortResults' => 1,
+		'tests' => [
+
+                        {
+                        'num' => 1,
+                	'floatpostprocess' => 1,
+                	'delimiter' => '	',
+                        'hive' => q\
+select 
+  a.intnum1000,
+  b.intnum1000,
+  a.id,
+  b.id,
+  a.intnum5,
+  b.intnum5,
+  a.intnum100,
+  b.intnum100,
+  a.intnum,
+  b.intnum,
+  a.longnum,
+  b.longnum,
+  a.floatnum,
+  b.floatnum,
+  a.doublenum,
+  b.doublenum
+from 
+       numbers a
+  join numbers b
+  on (a.intnum5 = b.intnum5)
+where
+      a.id < 30
+  and b.id < 40
+order by a.intnum5
+;
+\,
+                        'sql' => "
+select
+  a.intnum1000,
+  b.intnum1000,
+  a.id,
+  b.id,
+  a.intnum5,
+  b.intnum5,
+  a.intnum100,
+  b.intnum100,
+  a.intnum,
+  b.intnum,
+  a.longnum,
+  b.longnum,
+  a.floatnum,
+  b.floatnum,
+  a.doublenum,
+  b.doublenum
+from
+       numbers as a
+  join numbers as b
+  on (a.intnum5 = b.intnum5)
+where
+      a.id < 30
+  and b.id < 40
+order by a.intnum5
+;
+", 
+			},
+
+                        {
+                        'num' => 2,
+			# join by map entry 
+			# full outer join 
+                        'hive' => q\
+select
+  a.rownum,
+  b.rownum,
+  a.mymap['mymapk1'],
+  b.mymap['mymapk1'],
+  a.mymap['mymapk3'],
+  b.mymap['mymapk3'],
+  a.mymap,
+  b.mymap
+from
+                  complex a
+  full outer join complex b
+  on (a.mymap['mymapk1']
+    = b.mymap['mymapk3'])
+;
+\,
+                        'sql' => "\\d", # dummy statement
+			},
+
+                        {
+                        'num' => 3,
+			# join by tuple item 
+			# inner join 
+                        'hive' => q\
+select
+  a.rownum,
+  b.rownum,
+  a.mytuple.dbl,
+  b.mytuple.dbl,
+  a.mytuple,
+  b.mytuple
+from
+     complex a
+join complex b
+on (a.mytuple.dbl = b.mytuple.dbl + 1)
+;
+\,
+                        'sql' => "\\d", # dummy statement
+			},
+
+                        
+        		] # end tests
+        	}, # end group
+# end group
+#-------------------------------------------------------------------------------
+# HIVE STORED -> PIG SECTION
+#-------------------------------------------------------------------------------
+
+                {
+                'name' => 'hcat_hive2pig_load_describe',
+		# 
+
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'pig' => "
+a = load 'default.numbers_pig' using org.apache.hcatalog.pig.HCatLoader();
+describe a;
+",
+                        'rc' => 0, 
+			'expected_out' => 'a: {intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double}
+',
+			'not_expected_err_regex' => "ERROR",
+                        },
+
+                        {
+                        'num' => 2,
+                        'pig' => "
+a = load 'default.complex' using org.apache.hcatalog.pig.HCatLoader();
+describe a;
+",
+                        'rc' => 0, 
+			'expected_out' => 'a: {mymap: map[],mytuple: (num: int,str: chararray,dbl: double),bagofmap: {(innerfield: map[])},rownum: int}
+',
+			'not_expected_err_regex' => "ERROR",
+                        },
+
+                        {
+                        'num' => 3,
+			'ignore' => 'Pig does not understand boolean. Made a negative test for the error message.', 
+                        'pig' => "
+a = load 'default.boolean_table' using org.apache.hcatalog.pig.HCatLoader();
+describe a;
+",
+                        'rc' => 0, 
+			'expected_out' => "",
+			'not_expected_err_regex' => "ERROR",
+                        },
+
+                                                ] # end tests
+                }, # end group
+                {
+                'name' => 'hcat_hive2pig_type_check',
+		#
+ 		# * This UDF can be used to check that a tuple presented by org.apache.hcatalog.pig.HCatLoader has the
+ 		# * right types for the fields
+ 		# * (...)
+ 		# * The output should only contain the value '1' in all rows. (This UDF returns
+ 		# * the integer value 1 if all fields have the right type, else throws IOException)
+
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.numbers_pig' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('intnum1000:int,id:int,intnum5:int,intnum100:int,intnum:int,longnum:long,floatnum:float,doublenum:double', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+
+                        {
+                        'num' => 2,
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.complex' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('mymap: map[],mytuple: (num: int,str: chararray,dbl: double),bagofmap: {(innerfield: map[])},rownum: int', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+
+                        {
+                        'num' => 3,
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.boolean_table' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('boolean+int', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+
+                        
+                        ] # end tests
+                }, # end group
+                {
+                'name' => 'hcat_hive2pig_empty',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'hive' => "
+CREATE TABLE tmp_hive_empty_:RUNID: (
+  INTNUM1000      int,
+  ID              int,
+  INTNUM5         int,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_hive_empty_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                        {
+                        'num' => 2,
+			'depends_on' => 'hcat_hive2pig_empty_1',
+                        'pig' => "
+a = load 'default.tmp_hive_empty_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+store a into ':OUTPATH:';
+",
+                        'rc' => 0,
+                        'expected_err_regex' => "Success",
+                        'not_expected_err_regex' => "ERROR",
+                        'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 3,
+			'depends_on' => 'hcat_hive2pig_empty_1',
+                        'pig' => "
+a = load 'default.tmp_hive_empty_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+store a into 'default.tmp_hive_empty_:RUNID:'
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('',
+        'intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double'); 
+",
+                        'rc' => 0,
+                        'expected_err_regex' => "Success",
+                        'expected_out' => "",
+                        },
+
+                        ] # end tests
+                }, # end group
+        	{
+
+        	'name' => 'hcat_hive2pig_arithmetic_operators',
+		# covering any items from the "Arithmetic Operators and More" section not covered elswhere
+        	'delimiter' => '	',
+        	'tests' => [
+
+            		{
+            		'num' => 5,
+        		'sortResults' => 1,
+                	'sortBenchmark' => 1,
+			# COUNT star 
+            		'pig' => q?
+a = load 'default.numbers_pig' USING org.apache.hcatalog.pig.HCatLoader();
+b = group a all;
+c = foreach b generate COUNT(a.$0);
+store c into ':OUTPATH:';
+?,
+                        'rc' => 0,
+			'expected_err_regex' => "Success!",
+			'expected_stdout' => "5000",
+            		},
+            		{
+            		'num' => 6,
+			# Group
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+			# 'a: {intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double}'
+            		'pig' => q?
+a = load 'default.numbers_pig' USING org.apache.hcatalog.pig.HCatLoader();
+b = group a by intnum5;
+c = foreach b generate group as i5, COUNT(a) as count_rows, SUM(a.id) as sum_id;
+store c into ':OUTPATH:';
+?,
+                          'sql' => '
+select intnum5, COUNT(id), SUM(id)
+from numbers 
+group by intnum5
+;',
+            		},
+            		{
+            		'num' => 7,
+			# Order by
+			# 'a: {intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double}'
+			##########################################
+			#
+			# a = load '/user/hadoopqa/pig/tests/data/txt/numbers.txt' using PigStorage(':') 
+			#     as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
+			# b = group a by intnum5;
+			# c = foreach b generate group as i5, COUNT(a) as count_rows, SUM(a.id) as sum_id;
+			# d = order c by i5;
+			# dump d;
+			#
+			##########################################
+			#
+                        'floatpostprocess' => 1,
+                        # WE SHOULD REALLY NOT BE SORTING HERE, BUT WE CAN'T GET STABLE SORT OUT
+                        # OF PIG AND POSTGRES IN THE SAME MANNER
+			'sortBenchmark' => 1,
+			'sortResults' => 1,
+             		'pig' => q?
+a = load 'default.numbers_pig' USING org.apache.hcatalog.pig.HCatLoader();
+b = order a by intnum5;
+store b into ':OUTPATH:';
+?,
+                        #  'rc' => 0,
+			'sql' => 'select * from numbers order by intnum5 ;',
+            		},
+
+            		
+            		{
+            		'num' => 10,
+			# 9.3.5.1 FILTER such that an expression of the form:  - Part I
+			# FILTER alias by exp1
+			# FILTER alias by exp2
+			# FILTER alias by exp3
+			# gives the same result as
+			# 
+			# FILTER alias by exp1, exp2, expr3 
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'floatpostprocess' => 1,
+                        'delimiter' => '	',
+            		'pig' => q?
+a = load 'default.numbers_pig' USING org.apache.hcatalog.pig.HCatLoader();
+b = filter a by intnum1000 > 3000;
+c = filter b by id > 2500;
+d = filter c by intnum5 > 3;
+e = filter d by intnum > 5050;
+f = filter e by longnum > 5050;
+g = filter f by floatnum > 683;
+h = filter g by doublenum > 683;
+store h into ':OUTPATH:';
+?,
+            		'sql' => "
+select * from numbers
+where  
+    intnum1000 > 3000
+and id > 2500
+and intnum5 > 3
+and intnum > 5050
+and longnum > 5050
+and floatnum > 683
+and doublenum > 683
+;
+",
+            		},
+
+            		{
+            		'num' => 11,
+			# 9.3.5.1 FILTER such that an expression of the form:  - Part II
+			# FILTER alias by exp1
+			# FILTER alias by exp2
+			# FILTER alias by exp3
+			# gives the same result as
+			# 
+			# FILTER alias by exp1, exp2, expr3 
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'floatpostprocess' => 1,
+                        'delimiter' => '	',
+            		'pig' => q?
+a = load 'default.numbers_pig' USING org.apache.hcatalog.pig.HCatLoader();
+b = filter a by 
+          intnum1000 > 3000
+      and id > 2500
+      and intnum5 > 3
+      and intnum > 5050
+      and longnum > 5050
+      and floatnum > 683
+      and doublenum > 683;
+;
+store b into ':OUTPATH:';
+?,
+            		'sql' => "
+select * from numbers
+where  
+    intnum1000 > 3000
+and id > 2500
+and intnum5 > 3
+and intnum > 5050
+and longnum > 5050
+and floatnum > 683
+and doublenum > 683
+;
+",
+            		},
+        		] # end tests
+        	}, # end group
+
+#-------------------------------------------------------------------------------
+# HIVE STORED -> HADOOP SECTION
+#-------------------------------------------------------------------------------
+
+                {
+                'name' => 'hcat_hive2hadoop_read',
+		# From:
+                'sortResults' => 1,
+                'sortBenchmark' => 1,
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        # sum numbers
+			# Using doctored benchmark file due to rounding error, e.g.:
+			# actual line  : 5000    4500500 2949    416084  5050809 5050809 1083307.100     1083308.561
+			# expected line: 5000    4500500 2949    416084  5050809 5050809 1083310.000     1083308.561
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.SumNumbers thrift://:THRIFTSERVER: :OUTPATH: -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "
+select 
+  intnum1000,
+  sum (id),
+  sum (intnum5),
+  sum (intnum100),
+  sum (intnum),
+  sum (longnum),
+  sum (floatnum),
+  sum (doublenum)
+from numbers
+group by intnum1000
+order by intnum1000;
+",
+                        },
+
+                        ] # end tests
+                }, # end group
+
+                {
+                'name' => 'hcat_hive2hadoop_type_check', 
+		# From:
+                'sortResults' => 1,
+                'sortBenchmark' => 1,
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        # typedatacheck complex
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: complex 'map<string,string>+struct<num:int,str:string,dbl:double>+array<map<string,string>>+int' :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "\\d;", # dummy
+                        },
+
+                        {
+                        'num' => 2,
+                        # typedatacheck numbers
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: numbers int+smallint+tinyint+int+int+bigint+float+double :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select * from numbers;",
+                        },
+
+                        {
+                        'num' => 3,
+                        # typedatacheck boolean_table
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: boolean_table boolean+int :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select * from boolean_table;",
+                        },
+
+                        ] # end tests
+                }, # end group
+
+                {
+                'name' => 'hcat_hive2hadoop_write_numbers_nopart',
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'hive' => "
+drop table numbers_nopart_empty_initially;
+",
+			# does not give error even if the table does not exist
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 2,
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_nopart_1',
+                        'hive' => "
+CREATE TABLE numbers_nopart_empty_initially (
+  INTNUM1000      int,
+  ID              smallint,
+  INTNUM5         tinyint,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+stored as RCFile
+location ':TMP:/hcat_tmp_tables/numbers_nopart_empty_initially'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc'   => 0
+                        },
+
+                        {
+                        'num' => 3,
+                        # storenumbers
+        		# 'nopart' -> data is written to the numbers_nopart_empty_initially table.
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_nopart_2',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.StoreNumbers thrift://:THRIFTSERVER: nopart -libjars file://:HCAT_JAR:
+\,
+                        'rc' => 0,
+			'expected_out' => "",
+			'expected_err_regex' => "HDFS_BYTES_WRITTEN=\\d+",
+			'not_expected_err_regex' => "FAILED",
+                        },
+
+                        {
+                        'num' => 4,
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_nopart_3',
+                	'sortResults' => 1,
+                	'sortBenchmark' => 1,
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+			'hive' => "
+select 
+  intnum1000,
+  id,
+  intnum5,
+  intnum100,
+  intnum,
+  longnum,
+  floatnum,
+  doublenum
+from numbers_nopart_empty_initially;
+",
+			'sql' => "
+select 
+  10 + intnum1000,
+  10 + id,
+  10 + intnum5,
+  10 + intnum100,
+  10 + intnum,
+  10 + longnum,
+  10 + floatnum,
+  10 + doublenum
+from numbers; 
+",
+                        },
+
+ # pig cannot handle tinyint and smallint, hence create a version of the table with 'int' instead
+                        {
+                        'num' => 5,
+                        'hive' => "
+drop table numbers_nopart_pig_empty_initially;
+",
+			# does not give error even if the table does not exist
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 6,
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_nopart_5',
+                        'hive' => "
+CREATE TABLE numbers_nopart_pig_empty_initially (
+  INTNUM1000      int,
+  ID              int,
+  INTNUM5         int,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+stored as RCFile
+location ':TMP:/hcat_tmp_tables/numbers_nopart_pig_empty_initially'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc'   => 0
+                        },
+
+                        {
+                        'num' => 7,
+                        # storenumbers
+        		# 'nopart' -> data is written to the numbers_nopart_pig_empty_initially table.
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_nopart_6',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.StoreNumbers thrift://:THRIFTSERVER: nopart_pig -libjars file://:HCAT_JAR:
+\,
+                        'rc' => 0,
+			'expected_out' => "",
+			'expected_err_regex' => "HDFS_BYTES_WRITTEN=\\d+",
+			'not_expected_err_regex' => "FAILED",
+                        },
+
+                        {
+                        'num' => 8,
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_nopart_7',
+                	'sortResults' => 1,
+                	'sortBenchmark' => 1,
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+			'hive' => "
+select 
+  intnum1000,
+  id,
+  intnum5,
+  intnum100,
+  intnum,
+  longnum,
+  floatnum,
+  doublenum
+from numbers_nopart_pig_empty_initially;
+",
+			'sql' => "
+select 
+  10 + intnum1000,
+  10 + id,
+  10 + intnum5,
+  10 + intnum100,
+  10 + intnum,
+  10 + longnum,
+  10 + floatnum,
+  10 + doublenum
+from numbers; 
+",
+                        },
+
+                        ] # end tests
+                }, # end group
+
+                {
+                'name' => 'hcat_hive2hadoop_write_numbers_part',
+                'sortResults' => 1,
+                'sortBenchmark' => 1,
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+			'hive' => "
+drop table numbers_part_empty_initially;
+",
+			# does not give error even if the table does not exist
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 2,
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_part_1',
+                        'hive' => "
+CREATE TABLE numbers_part_empty_initially (
+  INTNUM1000      int,
+  ID              smallint,
+  INTNUM5         tinyint,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+partitioned by (
+  datestamp string
+)
+stored as RCFile
+location ':TMP:/hcat_tmp_tables/numbers_part_empty_initially'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc'   => 0
+                        },
+
+                        {
+                        'num' => 3,
+                        # storenumbers
+        		# 'part' -> data is written to datestamp = '20100101' partition of the numbers_part_empty_initially table.
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_part_2',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.StoreNumbers thrift://:THRIFTSERVER: part -libjars file://:HCAT_JAR:
+\,
+                        'rc' => 0,
+			'expected_out' => "",
+			'expected_err_regex' => "HDFS_BYTES_WRITTEN=\\d+",
+			'not_expected_err_regex' => "FAILED",
+                        },
+
+                        {
+                        'num' => 4,
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_part_3',
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                	'sortResults' => 1,
+                	'sortBenchmark' => 1,
+			'hive' => "
+select 
+  intnum1000,
+  id,
+  intnum5,
+  intnum100,
+  intnum,
+  longnum,
+  floatnum,
+  doublenum,
+  datestamp
+from numbers_part_empty_initially;
+",
+			'sql' => "
+select 
+  10 + intnum1000,
+  10 + id,
+  10 + intnum5,
+  10 + intnum100,
+  10 + intnum,
+  10 + longnum,
+  10 + floatnum,
+  10 + doublenum,
+  '20100101' 
+from numbers; 
+",
+                        },
+
+                        ] # end tests
+                }, # end group
+
+                {
+                'name' => 'hcat_hive2hadoop_write_complex_nopart',
+                'sortResults' => 1,
+                'sortBenchmark' => 1,
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+			'hive' => "
+drop table complex_nopart_empty_initially;
+",
+			# does not give error even if the table does not exist
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 2,
+			'depends_on' => 'hcat_hive2hadoop_write_complex_nopart_1',
+                        'hive' => "
+CREATE TABLE complex_nopart_empty_initially (
+  mymap    map<string,string>,
+  mytuple  struct<num:int,str:string,dbl:double>,
+  bagofmap array<Map<string,string>>,
+  rownum   int
+)
+stored as rcfile
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+); 
+",
+                        'rc'   => 0
+                        },
+
+                        {
+                        'num' => 3,
+                        # storecomplex 
+			'depends_on' => 'hcat_hive2hadoop_write_complex_nopart_2',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.StoreComplex thrift://:THRIFTSERVER: -libjars file://:HCAT_JAR:
+\,
+                        'rc' => 0,
+			'expected_out' => "",
+			'expected_err_regex' => "HDFS_BYTES_WRITTEN=\\d+",
+			'not_expected_err_regex' => "FAILED",
+                        },
+
+                        {
+                        'num' => 4,
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                	'sortResults' => 1,
+                	'sortBenchmark' => 1,
+			'depends_on' => 'hcat_hive2hadoop_write_complex_nopart_3',
+			'hive' => "
+select 
+  mymap,
+  mytuple,
+  bagofmap,
+  rownum
+from complex_nopart_empty_initially;
+",
+			'sql' => "\\d;", # dummy
+                        },
+
+                        ] # end tests
+                }, # end group
+
+
+
+
+################################################################################
+# PIG STORED DATA
+################################################################################
+#
+# The bootstrap creates tables and loads data using Hive.
+# Here tables with identical schemas are created and data stored to them using Pig.
+# The tables are then verified and used by each of pig, hive, and hadoop.
+#
+# NOTE: The hcat_pig2pig_setup_tables group must run before the rest of the 
+# "pig stored data" groups.
+#
+################################################################################
+
+#-------------------------------------------------------------------------------
+# Prepare test tables using PIG + other 'store' tests
+#-------------------------------------------------------------------------------
+
+ # Currently loading from txt "using PigStorage(':') ..."
+
+                {
+                'name' => 'hcat_pig2pig_setup_tables',
+		# From:
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        # numbers stored by pig
+                        'hive' => "
+CREATE TABLE tmp_pig2pig_stored_numbers_:RUNID: (
+  INTNUM1000      int,
+  ID              int, -- smallint,
+  INTNUM5         int, -- tinyint,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_pig2pig_stored_numbers_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                        {
+                        'num' => 2, # GOOD
+			'depends_on' => 'hcat_pig2pig_setup_tables_1',
+			# currently not loading from Hive
+                        'pig' => q\
+A = load ':INPATH:/txt/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double); 
+store A into 'default.tmp_pig2pig_stored_numbers_:RUNID:' 
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('', 
+        'intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double'); 
+\,
+                        'rc' => 0,
+                        },
+
+                        {
+                        'num' => 3,
+                        # complex stored by pig
+			'depends_on' => 'hcat_pig2pig_setup_tables_2', # not really, but #4 does
+                        'hive' => "
+CREATE TABLE tmp_pig2pig_stored_complex_:RUNID: (
+  mymap       map<string, string>,
+  mytuple     struct<num:int,str:string,dbl:double>,
+  bagofmap    array<map<string,string>>,
+  rownum      int
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_pig2pig_stored_complex_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                        {
+                        'num' => 4,
+			'depends_on' => 'hcat_pig2pig_setup_tables_3',
+			# currently not loading from Hive
+                        'pig' => q\
+A = load 'default.complex' using org.apache.hcatalog.pig.HCatLoader();
+store A into 'default.tmp_pig2pig_stored_complex_:RUNID:' 
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('', 
+	'mymap: map[],mytuple: (num: int,str: chararray,dbl: double),bagofmap: {(innerfield: map[])},rownum: int');
+\,
+                        'rc' => 0,
+                        },
+
+                        {
+                        'num' => 5,
+                        # boolean_table stored by pig
+                        'hive' => "
+CREATE TABLE tmp_pig2pig_stored_boolean_table_:RUNID: (
+  myflag boolean,
+  rownum int
+)
+row format DELIMITED FIELDS TERMINATED BY '\001'
+           COLLECTION ITEMS TERMINATED BY '\002'
+           MAP KEYS TERMINATED BY '\003'
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_pig2pig_stored_boolean_table_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                        {
+                        'num' => 6,
+			'depends_on' => 'hcat_pig2pig_setup_tables_5',
+			'ignore' => 'Pig does not understnad boolean',
+                        'pig' => q\
+A = load 'default.boolean_table' using org.apache.hcatalog.pig.HCatLoader();
+store A into 'default.tmp_pig2pig_stored_boolean_table_:RUNID:' 
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('', 
+        'boolean:string,rownum:integer'); 
+\,
+                        'rc' => 0,
+                        },
+
+                                                ] # end tests
+
+                }, # end group
+
+                {
+                'name' => 'hcat_pig2pig_more_store',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'hive' => "
+CREATE TABLE tmp_pig2pig_store_table_1_:RUNID: (
+  INTNUM1000      int,
+  ID              int, -- smallint,
+  INTNUM5         int, -- tinyint,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_pig2pig_store_table_1_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                        {
+                        'num' => 2,
+			'depends_on' => 'hcat_pig2pig_more_store_1',
+                        'hive' => "
+CREATE TABLE tmp_pig2pig_store_table_2_:RUNID: (
+  INTNUM1000      int,
+  ID              int, -- smallint,
+  INTNUM5         int, -- tinyint,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_pig2pig_store_table_2_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                        {
+                        'num' => 3,
+			'depends_on' => 'hcat_pig2pig_more_store_2',
+                        'pig' => q\
+A = load ':INPATH:/txt/numbers.txt' using PigStorage(':') as (intnum1000,id,intnum5,intnum100,intnum,longnum,floatnum,doublenum);
+-- A = load 'default.numbers_pig' using org.apache.hcatalog.pig.HCatLoader();
+B = foreach A generate (int)intnum1000, (int)id, (int)intnum5, (int)intnum100, (int)intnum, (long)longnum, (float)floatnum, (double)doublenum; 
+C = filter B by id <  2000;
+D = filter B by id >= 2000;
+-- store to 1st table
+store C into 'default.tmp_pig2pig_store_table_1_:RUNID:' 
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('', 
+        'intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double'); 
+-- store to 2nd table
+store D into 'default.tmp_pig2pig_store_table_2_:RUNID:' 
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('', 
+        'intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double'); 
+\,
+                        'rc' => 0,
+			'expected_err_regex' => "Success",
+                        },
+
+                        {
+                        'num' => 4,
+                        # to verify data in two tables written above
+			'depends_on' => 'hcat_pig2pig_more_store_3',
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'hive' => "
+select id, intnum from tmp_pig2pig_store_table_1_:RUNID:
+",
+                        'sql' => "
+select id, intnum 
+from numbers
+where id < 2000
+;
+", 
+                        },
+
+                        {
+                        'num' => 5,
+                        # to verify data in two tables written above
+			'depends_on' => 'hcat_pig2pig_more_store_3',
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'hive' => "
+select id, intnum from tmp_pig2pig_store_table_2_:RUNID:
+",
+                        'sql' => "
+select id, intnum 
+from numbers
+where id >= 2000
+;
+", 
+                        },
+
+# ADD HCatStorer tests for:
+# * not specifying schema
+#   - schema from HCatLoad ===> DONE
+#   - schema from load with PigStorage and 'as'  ===> DONE
+#   - schema from 'generate' with types  ===> DONE
+# * not specifying anything 
+#   - as above plus non-partitioned
+# * adding columns
+# ERROR tests:
+# * missing columns
+                        {
+                        'num' => 6,
+                        # setup step
+                        'hive' => "
+CREATE TABLE tmp_pig2pig_store_numbers_partitioned_:RUNID: (
+  INTNUM1000   int,
+  ID           int, --smallint,
+  INTNUM5      int, --tinyint,
+  INTNUM100    int,
+  INTNUM       int,
+  LONGNUM      bigint,
+  FLOATNUM     float,
+  DOUBLENUM    double
+)
+partitioned by (
+  STR1	string,
+  STR2	string
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/pig2pig_store_numbers_partitioned_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+
+                                                {
+                        'num' => 9,
+			'depends_on' => 'hcat_pig2pig_more_store_6',
+			# Store into a new partition with org.apache.hcatalog.pig.HCatStorer 
+			# Store without specifying schema, schema from PigStorage 'as'
+                        'pig' => q\
+a = load ':INPATH:/txt/numbers.txt' using PigStorage(':') 
+    as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
+store a into 'default.tmp_pig2pig_store_numbers_partitioned_:RUNID:'
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('STR1=str1part1,STR2=str2part1'); 
+\,
+                        'rc' => 0,
+                        },
+
+                        {
+                        'num' => 10,
+			'depends_on' => 'hcat_pig2pig_more_store_9',
+			# verify the above
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'pig' => q\
+a = load 'default.tmp_pig2pig_store_numbers_partitioned_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+b = filter a BY (
+           str1 == 'str1part1' and
+           str2 == 'str2part1'
+    );
+store b into ':OUTPATH:';
+\,
+                        'sql' => q\
+select *, 'str1part1', 'str2part1' from numbers;
+\,
+                        },
+
+                        {
+                        'num' => 11,
+			'depends_on' => 'hcat_pig2pig_more_store_6',
+			# Store into a new partition with org.apache.hcatalog.pig.HCatStorer 
+			# Store without specifying schema, schema from PigStorage 'as'
+                        'pig' => q\
+a = load ':INPATH:/txt/numbers.txt' using PigStorage(':') 
+    as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
+store a into 'default.tmp_pig2pig_store_numbers_partitioned_:RUNID:'
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('STR1=str1part2,STR2=str2part2'); 
+\,
+                        'rc' => 0,
+                        },
+
+                        {
+                        'num' => 12,
+			'depends_on' => 'hcat_pig2pig_more_store_11',
+			# verify the above
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'pig' => q\
+a = load 'default.tmp_pig2pig_store_numbers_partitioned_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+b = filter a BY (
+           str1 == 'str1part1' and
+           str2 == 'str2part1'
+    );
+store b into ':OUTPATH:';
+\,
+                        'sql' => q\
+select *, 'str1part1', 'str2part1' from numbers;
+\,
+                        },
+
+                        {
+                        'num' => 13,
+			'depends_on' => 'hcat_pig2pig_more_store_6',
+			# Store into a new partition with org.apache.hcatalog.pig.HCatStorer 
+			# Store without specifying schema, schema from 'generate' with types
+                        'pig' => q\
+a = load ':INPATH:/txt/numbers.txt' using PigStorage(':') as (intnum1000, id, intnum5, intnum100, intnum, longnum, floatnum, doublenum);
+b = foreach a generate (int)intnum1000, (int)id, (int)intnum5, (int)intnum100, (int)intnum, (long)longnum, (float)floatnum, (double)doublenum; 
+store b into 'default.tmp_pig2pig_store_numbers_partitioned_:RUNID:'
+        using org.apache.hcatalog.pig.HCatStorer 
+       ('STR1=str1part3,STR2=str2part3'); 
+\,
+                        'rc' => 0,
+                        },
+
+                        {
+                        'num' => 14,
+			'depends_on' => 'hcat_pig2pig_more_store_13',
+			# verify the above
+                	'sortBenchmark' => 1,
+                	'sortResults' => 1,
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'pig' => q\
+a = load 'default.tmp_pig2pig_store_numbers_partitioned_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+b = filter a BY (
+           str1 == 'str1part2' and
+           str2 == 'str2part2'
+    );
+store b into ':OUTPATH:';
+\,
+                        'sql' => q\
+select *, 'str1part2', 'str2part2' from numbers;
+\,
+                        },
+
+
+                        ] # end tests
+
+                }, # end group
+
+#-------------------------------------------------------------------------------
+# PIG STORED -> HIVE SECTION
+# Not a likely use case, commented out for time being
+#-------------------------------------------------------------------------------
+
+ #NaUC# 		{
+ #NaUC# 		'name' => 'hcat_pig2hive_type_check',
+ #NaUC# 		'tests' => [
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 1,
+ #NaUC# 			'hive' => q\ 
+ #NaUC# add jar :FUNCPATH:/testudf.jar;
+ #NaUC# create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+ #NaUC# select typecheck('int+smallint+tinyint+int+int+bigint+float+double',
+ #NaUC# intnum1000,id,intnum5,intnum100,intnum,longnum,floatnum,doublenum) from tmp_pig2pig_stored_numbers_:RUNID:;
+ #NaUC# \,
+ #NaUC# 			'rc' => 0,
+ #NaUC# 			'expected_out_regex' => "class java",
+ #NaUC# 			'expected_err_regex' => "OK",
+ #NaUC# 			'not_expected_err_regex' => "FAILED",
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 2,
+ #NaUC# 			'hive' => q\ 
+ #NaUC# add jar :FUNCPATH:/testudf.jar;
+ #NaUC# create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+ #NaUC# select typecheck('map<string,string>+struct<num:int,str:string,dbl:double>+array<map<string,string>>+int',
+ #NaUC# mymap, mytuple, bagofmap, rownum) from tmp_pig2pig_stored_complex_:RUNID:;
+ #NaUC# \,
+ #NaUC# 			'rc' => 0,
+ #NaUC# 			'expected_out_regex' => "class java",
+ #NaUC# 			'expected_err_regex' => "OK",
+ #NaUC# 			'not_expected_err_regex' => "FAILED",
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 3,
+ #NaUC# 			'hive' => q\ 
+ #NaUC# add jar :FUNCPATH:/testudf.jar;
+ #NaUC# create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+ #NaUC# select typecheck('boolean+int',
+ #NaUC# myflag,rownum) from tmp_pig2pig_stored_boolean_table_:RUNID:;
+ #NaUC# \,
+ #NaUC# 			'rc' => 0,
+ #NaUC# 			'expected_out_regex' => "class java",
+ #NaUC# 			'expected_err_regex' => "OK",
+ #NaUC# 			'not_expected_err_regex' => "FAILED",
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 4,
+ #NaUC# 			'hive' => q\ 
+ #NaUC# add jar :FUNCPATH:/testudf.jar;
+ #NaUC# create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+ #NaUC# select typecheck('string+string+string+string+int+string+string+string+string+string+string+string+string+string+map<string,string>+map<string,string>+array<map<string,string>>+string+string+string+string',
+ #NaUC# bcookie,
+ #NaUC# src_spaceid,
+ #NaUC# srcpvid,
+ #NaUC# ts,
+ #NaUC# browser,
+ #NaUC# bckt,
+ #NaUC# type,
+ #NaUC# ip,
+ #NaUC# yuid,
+ #NaUC# referrer,
+ #NaUC# pg_spaceid,
+ #NaUC# dstid,
+ #NaUC# dstpvid,
+ #NaUC# dst_spaceid,
+ #NaUC# page_params,
+ #NaUC# clickinfo,
+ #NaUC# viewinfo,
+ #NaUC# datestamp,
+ #NaUC# srcid,
+ #NaUC# action,
+ #NaUC# testid
+ #NaUC# )
+ #NaUC# where
+ #NaUC#     datestamp = '20091102'
+ #NaUC# and srcid     = '19174'
+ #NaUC# and browser   = 3
+ #NaUC# and src_spaceid = '2114728002'
+ #NaUC# limit 15
+ #NaUC# ;
+ #NaUC# \,
+ #NaUC# 			'rc' => 0,
+ #NaUC# 			'expected_out_regex' => "class java",
+ #NaUC# 			'expected_err_regex' => "OK",
+ #NaUC# 			'not_expected_err_regex' => "FAILED",
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 5,
+ #NaUC#                         'floatpostprocess' => 1,
+ #NaUC#                         'delimiter' => '	',
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   id, -- expect smallint
+ #NaUC#   intnum5 + id, -- expect smallint
+ #NaUC#   intnum5 + id + intnum, -- expect int
+ #NaUC#   intnum5 + id + intnum + longnum, -- expect bigint
+ #NaUC#   intnum5 * id, -- expect smallint
+ #NaUC#   intnum5 * id * intnum, -- expect int
+ #NaUC#   intnum5 * id * intnum * longnum, -- expect bigint
+ #NaUC#   intnum5 + 500, -- expect int
+ #NaUC#   intnum5 + 1.5, -- expect float
+ #NaUC#   cast(intnum5 + 1.5 as double), -- expect double
+ #NaUC#   cast(intnum5 + 1.0 as int), -- expect int
+ #NaUC#   floatnum + doublenum, -- expect double
+ #NaUC#   floatnum * doublenum -- expect double
+ #NaUC# from tmp_pig2pig_stored_numbers_:RUNID:
+ #NaUC# order by id
+ #NaUC# limit 500
+ #NaUC# ;
+ #NaUC# \,
+ #NaUC#                         'sql' => "
+ #NaUC# select
+ #NaUC#   id,
+ #NaUC#   intnum5 + id,
+ #NaUC#   intnum5 + id + intnum,
+ #NaUC#   intnum5 + id + intnum + longnum,
+ #NaUC#   intnum5 * id,
+ #NaUC#   intnum5 * id * intnum,
+ #NaUC#   intnum5 * id * intnum * longnum,
+ #NaUC#   intnum5 + 500,
+ #NaUC#   intnum5 + 1.5,
+ #NaUC#   cast(intnum5 + 1.5 as double precision),
+ #NaUC#   cast(intnum5 + 1.0 as integer),
+ #NaUC#   floatnum + doublenum,
+ #NaUC#   floatnum * doublenum
+ #NaUC# from numbers
+ #NaUC# order by id
+ #NaUC# limit 500
+ #NaUC# ;
+ #NaUC# ",
+ #NaUC#                         },
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 6,
+ #NaUC#                         'ignore' => '!!! Hive truncates where Postgres rounds',
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   intnum5, 
+ #NaUC#   cast(intnum5 + 1.9 as int),
+ #NaUC# from tmp_pig2pig_stored_numbers_:RUNID:
+ #NaUC# order by id
+ #NaUC# limit 5
+ #NaUC# ;
+ #NaUC# \,
+ #NaUC#                         'sql' => "
+ #NaUC# select
+ #NaUC#   intnum5, 
+ #NaUC#   cast(intnum5 + 1.9 as integer), 
+ #NaUC# from numbers
+ #NaUC# order by id
+ #NaUC# limit 5
+ #NaUC# ;
+ #NaUC# ", 
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#         		] # end tests
+ #NaUC#         	}, # end group
+ #NaUC# 
+ #NaUC# 		{
+ #NaUC# 		'name' => 'hcat_pig2hive_diagnostic_operators',
+ #NaUC# 		'tests' => [
+ #NaUC# 
+ #NaUC# 			 #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 5,
+ #NaUC# 			'hive' => "describe tmp_pig2pig_stored_complex_:RUNID:;",
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 6,
+ #NaUC#                         'ignore' => 'Open question about expected output',
+ #NaUC# 			'hive' => "describe tmp_pig2pig_stored_complex_:RUNID:.mytuple;",
+ #NaUC# 			'rc' => 0,
+ #NaUC# 			'expected_out_regex' => "mytuple\tstruct<num:int,str:string,dbl:double>\tfrom deserializer",
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 7,
+ #NaUC# 			'hive' => "describe tmp_pig2pig_stored_complex_:RUNID:.bagofmap;",
+ #NaUC# 			'rc' => 0,
+ #NaUC# 			'expected_out_regex' => "bagofmap\tarray<map<string,string>>\tfrom deserializer",
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 8,
+ #NaUC# 			'hive' => "describe tmp_pig2pig_stored_boolean_table_:RUNID:;",
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC# 			{
+ #NaUC# 			'num' => 9,
+ #NaUC# 			'hive' => "describe tmp_pig2pig_stored_boolean_table_:RUNID:.myflag;",
+ #NaUC# 			'rc' => 0,
+ #NaUC# 			'expected_out_regex' => "myflag\tboolean\tfrom deserializer",
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#         		] # end tests
+ #NaUC#         	}, # end group
+ #NaUC# 
+ #NaUC# 		{
+ #NaUC# 		'name' => 'hcat_pig2hive_select',
+ #NaUC#                 'sortBenchmark' => 1,
+ #NaUC#                 'sortResults' => 1,
+ #NaUC# 		'tests' => [
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 1,
+ #NaUC#                 	'floatpostprocess' => 1,
+ #NaUC#                 	'delimiter' => '	',
+ #NaUC# 			# actually no nulls in table, so had not needed the casting..., 
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   if (intnum1000 IS NULL, "", cast(intnum1000 as STRING)),
+ #NaUC#   if (id IS NULL, "", cast(id as STRING)),
+ #NaUC#   if (intnum5 IS NULL, "", cast(intnum5 as STRING)),
+ #NaUC#   if (intnum100 IS NULL, "", cast(intnum100 as STRING)),
+ #NaUC#   if (intnum IS NULL, "", cast(intnum as STRING)),
+ #NaUC#   if (longnum IS NULL, "", cast(longnum as STRING)),
+ #NaUC#   if (floatnum IS NULL, "", cast(floatnum as STRING)),
+ #NaUC#   if (doublenum IS NULL, "", cast(doublenum as STRING))
+ #NaUC# from tmp_pig2pig_stored_numbers_:RUNID:;
+ #NaUC# \,
+ #NaUC#                         'sql' => q\
+ #NaUC# select
+ #NaUC#   intnum1000,
+ #NaUC#   id,
+ #NaUC#   intnum5,
+ #NaUC#   intnum100,
+ #NaUC#   intnum,
+ #NaUC#   longnum,
+ #NaUC#   floatnum,
+ #NaUC#   doublenum
+ #NaUC# from numbers;
+ #NaUC# \,
+ #NaUC#                         },
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 2,
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   mymap,
+ #NaUC#   mytuple,
+ #NaUC#   bagofmap,
+ #NaUC#   rownum
+ #NaUC# from tmp_pig2pig_stored_complex_:RUNID:;
+ #NaUC# \,
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC#                         },
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 3,
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   myflag, rownum
+ #NaUC# from tmp_pig2pig_stored_boolean_table_:RUNID:;
+ #NaUC# \,
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC#                         },
+ #NaUC# 
+ #NaUC# 			 #NaUC# 			 #NaUC# 
+ #NaUC# 			 #NaUC# 
+ #NaUC# 			 #NaUC#                         {
+ #NaUC#                         'num' => 8,
+ #NaUC#                 	'sortBenchmark' => 1,
+ #NaUC# 	                'sortResults' => 1,
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   distinct 
+ #NaUC#   intnum1000,
+ #NaUC#   intnum5,
+ #NaUC#   intnum100,
+ #NaUC#   intnum,
+ #NaUC#   longnum
+ #NaUC# from tmp_pig2pig_stored_numbers_:RUNID:;
+ #NaUC# \,
+ #NaUC#                         'sql' => q\
+ #NaUC# select
+ #NaUC#   distinct 
+ #NaUC#   intnum1000,
+ #NaUC#   intnum5,
+ #NaUC#   intnum100,
+ #NaUC#   intnum,
+ #NaUC#   longnum
+ #NaUC# from numbers;
+ #NaUC# \,
+ #NaUC#                         },
+ #NaUC# 
+ #NaUC#         		] # end tests
+ #NaUC#         	}, # end group
+ #NaUC# 
+ #NaUC# 		{
+ #NaUC# 		'name' => 'hcat_pig2hive_lateral',
+ #NaUC#                 'sortBenchmark' => 1,
+ #NaUC#                 'sortResults' => 1,
+ #NaUC# 		'tests' => [
+ #NaUC# 
+ #NaUC# 			# NOTE: 
+ #NaUC# 			# The queries below written w/o WHERE clauses until this jira is resolved:
+ #NaUC# 			# https://issues.apache.org/jira/browse/HIVE-1056
+ #NaUC# 			# http://wiki.apache.org/hadoop/Hive/LanguageManual/LateralView
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 1,
+ #NaUC#                         'hive' => q\
+ #NaUC# select rownum, item 
+ #NaUC# from tmp_pig2pig_stored_complex_:RUNID: 
+ #NaUC#      LATERAL VIEW explode(bagofmap) explodedTable AS item;
+ #NaUC# \,
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#                          #NaUC#                         {
+ #NaUC#                         'num' => 3,
+ #NaUC#                         'hive' => q\
+ #NaUC# select 
+ #NaUC#   rownum, 
+ #NaUC#   item, 
+ #NaUC#   "item['a1']:", if (item['a1'] IS NULL, "", cast(item['a1'] as STRING)),
+ #NaUC#   "item['a2']:", if (item['a2'] IS NULL, "", cast(item['a2'] as STRING)),
+ #NaUC#   "item['k1']:", if (item['k1'] IS NULL, "", cast(item['k1'] as STRING)),
+ #NaUC#   "item['k6']:", if (item['k6'] IS NULL, "", cast(item['k6'] as STRING))
+ #NaUC# from complex LATERAL VIEW explode(bagofmap) explodedTable AS item
+ #NaUC# ;
+ #NaUC# \,
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#                          #NaUC# 
+ #NaUC#         		] # end tests
+ #NaUC#         	}, # end group
+ #NaUC# 
+ #NaUC# 		{
+ #NaUC# 		'name' => 'hcat_pig2hive_join',
+ #NaUC#                 'sortBenchmark' => 1,
+ #NaUC#                 'sortResults' => 1,
+ #NaUC# 		'tests' => [
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 1,
+ #NaUC#                 	'floatpostprocess' => 1,
+ #NaUC#                 	'delimiter' => '	',
+ #NaUC#                         'hive' => q\
+ #NaUC# select 
+ #NaUC#   a.intnum1000,
+ #NaUC#   b.intnum1000,
+ #NaUC#   a.id,
+ #NaUC#   b.id,
+ #NaUC#   a.intnum5,
+ #NaUC#   b.intnum5,
+ #NaUC#   a.intnum100,
+ #NaUC#   b.intnum100,
+ #NaUC#   a.intnum,
+ #NaUC#   b.intnum,
+ #NaUC#   a.longnum,
+ #NaUC#   b.longnum,
+ #NaUC#   a.floatnum,
+ #NaUC#   b.floatnum,
+ #NaUC#   a.doublenum,
+ #NaUC#   b.doublenum
+ #NaUC# from 
+ #NaUC#        tmp_pig2pig_stored_numbers_:RUNID: a
+ #NaUC#   join numbers b
+ #NaUC#   on (a.intnum5 = b.intnum5)
+ #NaUC# where
+ #NaUC#       a.id < 30
+ #NaUC#   and b.id < 40
+ #NaUC# order by a.intnum5
+ #NaUC# ;
+ #NaUC# \,
+ #NaUC#                         'sql' => "
+ #NaUC# select
+ #NaUC#   a.intnum1000,
+ #NaUC#   b.intnum1000,
+ #NaUC#   a.id,
+ #NaUC#   b.id,
+ #NaUC#   a.intnum5,
+ #NaUC#   b.intnum5,
+ #NaUC#   a.intnum100,
+ #NaUC#   b.intnum100,
+ #NaUC#   a.intnum,
+ #NaUC#   b.intnum,
+ #NaUC#   a.longnum,
+ #NaUC#   b.longnum,
+ #NaUC#   a.floatnum,
+ #NaUC#   b.floatnum,
+ #NaUC#   a.doublenum,
+ #NaUC#   b.doublenum
+ #NaUC# from
+ #NaUC#        numbers as a
+ #NaUC#   join numbers as b
+ #NaUC#   on (a.intnum5 = b.intnum5)
+ #NaUC# where
+ #NaUC#       a.id < 30
+ #NaUC#   and b.id < 40
+ #NaUC# order by a.intnum5
+ #NaUC# ;
+ #NaUC# ", 
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 2,
+ #NaUC# 			# join by map entry 
+ #NaUC# 			# full outer join 
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   a.rownum,
+ #NaUC#   b.rownum,
+ #NaUC#   a.mymap['mymapk1'],
+ #NaUC#   b.mymap['mymapk1'],
+ #NaUC#   a.mymap['mymapk3'],
+ #NaUC#   b.mymap['mymapk3'],
+ #NaUC#   a.mymap,
+ #NaUC#   b.mymap
+ #NaUC# from
+ #NaUC#                   tmp_pig2pig_stored_complex_:RUNID: a
+ #NaUC#   full outer join complex b
+ #NaUC#   on (a.mymap['mymapk1']
+ #NaUC#     = b.mymap['mymapk3'])
+ #NaUC# ;
+ #NaUC# \,
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#                         {
+ #NaUC#                         'num' => 3,
+ #NaUC# 			# join by tuple item 
+ #NaUC# 			# inner join 
+ #NaUC#                         'hive' => q\
+ #NaUC# select
+ #NaUC#   a.rownum,
+ #NaUC#   b.rownum,
+ #NaUC#   a.mytuple.dbl,
+ #NaUC#   b.mytuple.dbl,
+ #NaUC#   a.mytuple,
+ #NaUC#   b.mytuple
+ #NaUC# from
+ #NaUC#      tmp_pig2pig_stored_complex_:RUNID: a
+ #NaUC# join complex b
+ #NaUC# on (a.mytuple.dbl = b.mytuple.dbl + 1)
+ #NaUC# ;
+ #NaUC# \,
+ #NaUC#                         'sql' => "\\d", # dummy statement
+ #NaUC# 			},
+ #NaUC# 
+ #NaUC#         		] # end tests
+ #NaUC#         	}, # end group
+ #NaUC# 
+ #NaUC# 		
+#-------------------------------------------------------------------------------
+# PIG STORED -> PIG SECTION
+#-------------------------------------------------------------------------------
+
+                {
+                'name' => 'hcat_pig2pig_type_check',
+		# Using udf from:
+		#
+ 		# * This UDF can be used to check that a tuple presented by org.apache.hcatalog.pig.HCatLoader has the
+ 		# * right types for the fields
+ 		# * (...)
+ 		# * The output should only contain the value '1' in all rows. (This UDF returns
+ 		# * the integer value 1 if all fields have the right type, else throws IOException)
+
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.tmp_pig2pig_stored_numbers_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('intnum1000:int,id:int,intnum5:int,intnum100:int,intnum:int,longnum:long,floatnum:float,doublenum:double', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+
+                        {
+                        'num' => 2,
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.tmp_pig2pig_stored_complex_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+-- b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('{mymap: map[],mytuple: (num: int,str: chararray,dbl: double),bagofmap: {(innerfield: map[])},rownum: int', *);
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('mymap: map[],mytuple: (num: int,str: chararray,dbl: double),bagofmap: {(innerfield: map[])},rownum: int', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+
+                        {
+                        'num' => 3,
+			'ignore' => 'pig does not understand boolean',
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.tmp_pig2pig_stored_boolean_table_:RUNID:' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('boolean+int', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+                        
+                        ] # end tests
+                }, # end group
+
+#-------------------------------------------------------------------------------
+# PIG STORED -> HADOOP SECTION
+#-------------------------------------------------------------------------------
+
+                {
+                'name' => 'hcat_pig2hadoop_read',
+                'sortResults' => 1,
+                'sortBenchmark' => 1,
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+ # This tests is covered under hcat_hive2hadoop_read_1
+ #                        {
+ #                        'num' => 1,
+ #                        # sum numbers
+ #                        'ignore' => '"sumnumbers" is hard coded to run on "numbers", which is created by Hive, not pig.',
+ #                        'floatpostprocess' => 0,
+ #                        'delimiter' => '	',
+ #                        'hadoop' => q\
+ #jar :FUNCPATH:/testudf.jar sumnumbers thrift://:THRIFTSERVER: :OUTPATH: -libjars file://:HCAT_JAR:
+ #\,
+ #                        'sql' => "
+ #select
+ #  intnum1000,
+ #  sum (id),
+ #  sum (intnum5),
+ #  sum (intnum100),
+ #  sum (intnum),
+ #  sum (longnum),
+ #  sum (floatnum),
+ #  sum (doublenum)
+ #from numbers
+ #group by intnum1000
+ #order by intnum1000;
+ #",
+ #                        },
+
+                        ] # end tests
+                }, # end group
+
+                {
+                'name' => 'hcat_pig2hadoop_type_check', 
+                'sortResults' => 1,
+                'sortBenchmark' => 1,
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        # typedatacheck complex
+# aborts !!!
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: tmp_pig2pig_stored_complex_:RUNID: 'map<string,string>+struct<num:int,str:string,dbl:double>+array<map<string,string>>+int' :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select * from complex;",
+                        },
+
+                        {
+                        'num' => 2,
+                        # typedatacheck numbers
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: tmp_pig2pig_stored_numbers_:RUNID: int+int+int+int+int+bigint+float+double :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select * from numbers;",
+                        },
+
+                        {
+                        'num' => 3,
+                        # typedatacheck boolean_table
+                        'ignore' => 'pig does not understand boolean',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: tmp_pig2pig_stored_boolean_table_:RUNID: boolean+int :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select * from boolean_table;",
+                        },
+
+                        ] # end tests
+                }, # end group
+
+
+################################################################################
+# HADOOP STORED DATA
+################################################################################
+#
+# The bootstrap creates tables and loads data using Hive.
+# Here tables with identical schemas are created and data stored to them using Hadoop.
+# The tables are then verified and used by each of pig, hive, and hadoop.
+#
+# NOTE: The hcat_hive2hadoop_setup_tables group must run before the rest of the 
+# "hadoop stored data" groups.
+#
+################################################################################
+
+#-------------------------------------------------------------------------------
+# Prepare test tables using Hadoop
+#-------------------------------------------------------------------------------
+
+# This replicates the drop, create and writing parts from the hcat_hive2hadoop_... tests.
+# This should ideally be moved to the bootstrap conf, but would require a change in table name.
+
+                {
+                'name' => 'hcat_hive2hadoop_setup_tables',
+                'sortResults' => 1,
+                'sortBenchmark' => 1,
+                'hadoop_classpath' => ':HCAT_JAR:',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'hive' => "
+drop table numbers_nopart_empty_initially;
+",
+			# does not give error even if the table does not exist
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 2,
+			'depends_on' => 'hcat_hive2hadoop_setup_tables_1',
+                        'hive' => "
+CREATE TABLE numbers_nopart_empty_initially (
+  INTNUM1000      int,
+  ID              smallint,
+  INTNUM5         tinyint,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+stored as RCFile
+location ':TMP:/hcat_tmp_tables/numbers_nopart_empty_initially'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc'   => 0
+                        },
+
+                        {
+                        'num' => 3,
+                        # storenumbers
+        		# 'nopart' -> data is written to the numbers_nopart_empty_initially table.
+			'depends_on' => 'hcat_hive2hadoop_setup_tables_2',
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.StoreNumbers thrift://:THRIFTSERVER: nopart -libjars file://:HCAT_JAR:
+\,
+                        'rc' => 0,
+			'expected_out' => "",
+			'expected_err_regex' => "HDFS_BYTES_WRITTEN=\\d+",
+			'not_expected_err_regex' => "FAILED",
+                        },
+
+                        {
+                        'num' => 4,
+			'hive' => "
+drop table numbers_part_empty_initially;
+",
+			# does not give error even if the table does not exist
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 5,
+			'depends_on' => 'hcat_hive2hadoop_setup_tables_4',
+                        'hive' => "
+CREATE TABLE numbers_part_empty_initially (
+  INTNUM1000      int,
+  ID              smallint,
+  INTNUM5         tinyint,
+  INTNUM100       int,
+  INTNUM          int,
+  LONGNUM         bigint,
+  FLOATNUM        float,
+  DOUBLENUM       double
+)
+partitioned by (
+  datestamp string
+)
+stored as RCFile
+location ':TMP:/hcat_tmp_tables/numbers_part_empty_initially'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc'   => 0
+                        },
+
+                        {
+                        'num' => 6,
+                        # storenumbers
+        		# 'part' -> data is written to datestamp = '20100101' partition of the numbers_part_empty_initially table.
+			'depends_on' => 'hcat_hive2hadoop_setup_tables_5',
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.StoreNumbers thrift://:THRIFTSERVER: part -libjars file://:HCAT_JAR:
+\,
+                        'rc' => 0,
+			'expected_out' => "",
+			'expected_err_regex' => "HDFS_BYTES_WRITTEN=\\d+",
+			'not_expected_err_regex' => "FAILED",
+                        },
+
+                        {
+                        'num' => 7,
+			'hive' => "
+drop table complex_nopart_empty_initially;
+",
+			# does not give error even if the table does not exist
+			'expected_out' => "",
+                        },
+
+                        {
+                        'num' => 8,
+			'depends_on' => 'hcat_hive2hadoop_setup_tables_7',
+                        'hive' => "
+CREATE TABLE complex_nopart_empty_initially (
+  mymap    map<string,string>,
+  mytuple  struct<num:int,str:string,dbl:double>,
+  bagofmap array<Map<string,string>>,
+  rownum   int
+)
+stored as rcfile
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+); 
+",
+                        'rc'   => 0
+                        },
+
+                        {
+                        'num' => 9,
+                        # storecomplex 
+			'depends_on' => 'hcat_hive2hadoop_setup_tables_8',
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.StoreComplex thrift://:THRIFTSERVER: -libjars file://:HCAT_JAR:
+\,
+                        'rc' => 0,
+			'expected_out' => "",
+			'expected_err_regex' => "HDFS_BYTES_WRITTEN=",
+			'not_expected_err_regex' => "FAILED",
+                        },
+
+			# !!! Add tests reading the just stored data 
+
+                        ] # end tests
+                }, # end group
+
+#-------------------------------------------------------------------------------
+# HADOOP STORED -> HIVE SECTION
+#-------------------------------------------------------------------------------
+
+		{
+		'name' => 'hcat_hadoop2hive_type_check',
+		'tests' => [
+			# for how the 'typecheck' udf works.
+			# If the the typecheck does not pass, the udf will abort.
+			# Look at the jobtracker for info about the actual value, 
+			# e.g. something like: "Expected java.lang.String, got java.lang.Integer"
+			# The verification:
+			# 'expected_out_regex' => "class java",
+			# is needed because otherwise the tests might "PASS" because the output was empty...
+
+			{
+			'num' => 1,
+			'hive' => q\ 
+add jar :FUNCPATH:/testudf.jar;
+create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+select typecheck('int+smallint+tinyint+int+int+bigint+float+double',
+intnum1000,id,intnum5,intnum100,intnum,longnum,floatnum,doublenum) from numbers_nopart_empty_initially;
+\,
+			'rc' => 0,
+			'expected_out_regex' => "class java",
+			'expected_err_regex' => "OK",
+			'not_expected_err_regex' => "FAILED",
+			},
+
+			{
+			'num' => 2,
+			'depends_on' => 'hcat_hive2hadoop_setup_tables_6',
+			'hive' => q\ 
+add jar :FUNCPATH:/testudf.jar;
+create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+select typecheck('int+smallint+tinyint+int+int+bigint+float+double+string',
+intnum1000,id,intnum5,intnum100,intnum,longnum,floatnum,doublenum,datestamp) from numbers_part_empty_initially;
+\,
+			'rc' => 0,
+			'expected_out_regex' => "class java",
+			'expected_err_regex' => "OK",
+			'not_expected_err_regex' => "FAILED",
+			},
+
+			{
+			'num' => 3,
+			'hive' => q\ 
+add jar :FUNCPATH:/testudf.jar;
+create temporary function typecheck as 'org.apache.hcatalog.utils.HCatTypeCheckHive';
+select typecheck('map<string,string>+struct<num:int,str:string,dbl:double>+array<map<string,string>>+int',
+mymap, mytuple, bagofmap, rownum) from complex_nopart_empty_initially;
+\,
+			'rc' => 0,
+			'expected_out_regex' => "class java",
+			'expected_err_regex' => "OK",
+			'not_expected_err_regex' => "FAILED",
+			},
+
+                        ] # end tests
+                }, # end group
+
+#-------------------------------------------------------------------------------
+# HADOOP STORED -> PIG SECTION
+#-------------------------------------------------------------------------------
+
+                {
+                'name' => 'hcat_hadoop2pig_type_check',
+ 		# * This UDF can be used to check that a tuple presented by org.apache.hcatalog.pig.HCatLoader has the
+ 		# * right types for the fields
+ 		# * (...)
+ 		# * The output should only contain the value '1' in all rows. (This UDF returns
+ 		# * the integer value 1 if all fields have the right type, else throws IOException)
+
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.numbers_nopart_pig_empty_initially' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('intnum1000:int,id:int,intnum5:int,intnum100:int,intnum:int,longnum:long,floatnum:float,doublenum:double', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+
+ # default.numbers_part_empty_initially has smallint/tinyint which cannot be handled by pig
+ #                        {
+ #                        'num' => 2,
+ #                        'pig' => "
+ #register :FUNCPATH:/testudf.jar;
+ #a = load 'default.numbers_part_empty_initially' using org.apache.hcatalog.pig.HCatLoader();
+ #b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('intnum1000:int,id:int,intnum5:int,intnum100:int,intnum:int,longnum:long,floatnum:float,doublenum:double+string', *);
+ #store b into ':OUTPATH:';
+ #",
+ #                        'rc' => 0, 
+ #			'expected_err_regex' => "Success",
+ #			# 'expected_out_regex' => "1",
+ #			# 'not_expected_out_regex' => "[^1\\t]",
+ #                        },
+
+                        {
+                        'num' => 3,
+                        'pig' => "
+register :FUNCPATH:/testudf.jar;
+a = load 'default.complex_nopart_empty_initially' using org.apache.hcatalog.pig.HCatLoader();
+b = foreach a generate org.apache.hcatalog.utils.HCatTypeCheck('m:map[],t:tuple(num:int,str:chararray,dbl:double),bg:bag{t:tuple(m:map[])},i:int', *);
+store b into ':OUTPATH:';
+",
+                        'rc' => 0, 
+			'expected_err_regex' => "Success",
+			# 'expected_out_regex' => "1",
+			# 'not_expected_out_regex' => "[^1\\t]",
+                        },
+
+                        ] # end tests
+                }, # end group
+
+#-------------------------------------------------------------------------------
+# HADOOP STORED -> HADOOP SECTION
+#-------------------------------------------------------------------------------
+
+                {
+                'name' => 'hcat_hadoop2hadoop_type_check',
+ 		# * This UDF can be used to check that a tuple presented by org.apache.hcatalog.pig.HCatLoader has the
+ 		# * right types for the fields
+ 		# * (...)
+ 		# * The output should only contain the value '1' in all rows. (This UDF returns
+ 		# * the integer value 1 if all fields have the right type, else throws IOException)
+
+                'tests' => [
+
+                        {
+                        'num' => 1,
+                        # typedatacheck complex_nopart_empty_initially
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: complex_nopart_empty_initially 'map<string,string>+struct<num:int,str:string,dbl:double>+array<map<string,string>>+int' :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select * from complex;",
+                        },
+
+                        {
+                        'num' => 2,
+                        # typedatacheck numbers_nopart_empty_initially
+			'depends_on' => 'hcat_hive2hadoop_write_numbers_nopart_3',
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: numbers_nopart_empty_initially int+smallint+tinyint+int+int+bigint+float+double :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select intnum1000+10, id+10, intnum5+10, intnum100 + 10, intnum+10, longnum+10, floatnum+10, doublenum+10 from numbers;",
+                        },
+
+                        {
+                        'num' => 3,
+                        # typedatacheck numbers_part_empty_initially
+                        'floatpostprocess' => 0,
+                        'delimiter' => '	',
+                        'hadoop' => q\
+jar :FUNCPATH:/testudf.jar org.apache.hcatalog.utils.TypeDataCheck thrift://:THRIFTSERVER: numbers_part_empty_initially int+smallint+tinyint+int+int+bigint+float+double+string :OUTPATH: tab -libjars file://:HCAT_JAR:
+\,
+                        'sql' => "select intnum1000+10, id+10, intnum5+10, intnum100 + 10, intnum+10, longnum+10, floatnum+10, doublenum+10 , 20100101 from numbers;",
+                        },
+
+                        ] # end tests
+                }, # end group
+
+################################################################################
+# NEGATIVE
+################################################################################
+
+#-------------------------------------------------------------------------------
+# Negative: hive
+#-------------------------------------------------------------------------------
+
+                {
+                'name' => 'hcat_negative_hive',
+                'tests' => [
+
+                        {
+                        'num' => 1,
+			# Describe invalid Table 
+                        'hive' => "
+describe non_existing_table;
+",
+                        'rc' => 9, # what is expected ? !!!
+			'expected_err_regex' => "FAILED",
+			'not_expected_err_regex' => "OK",
+                        },
+
+                        {
+                        'num' => 2,
+                        # CREATE a table name that already exists, step 1
+                        'hive' => "
+CREATE TABLE tmp_name_collision_:RUNID: (
+  ID          int,
+  STR         string
+)
+stored as rcfile
+location ':TMP:/hcat_tmp_tables/tmp_name_collision_:RUNID:'
+TBLPROPERTIES (
+  'hcat.isd'='org.apache.hcatalog.rcfile.RCFileInputDriver',
+  'hcat.osd'='org.apache.hcatalog.rcfile.RCFileOutputDriver'
+);
+",
+                        'rc' => 0, 
+                        },
+

[... 259 lines stripped ...]


Mime
View raw message