hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1506394 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/ test/results/clientpositive/
Date Wed, 24 Jul 2013 04:02:00 GMT
Author: hashutosh
Date: Wed Jul 24 04:02:00 2013
New Revision: 1506394

URL: http://svn.apache.org/r1506394
Log:
HIVE-4547 : A complex create view statement fails with new Antlr 3.4 (Prasad Mujumdar via
Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q
    hive/trunk/ql/src/test/queries/clientpositive/view_cast.q
    hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out
    hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java?rev=1506394&r1=1506393&r2=1506394&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java Wed Jul
24 04:02:00 2013
@@ -94,61 +94,35 @@ class UnparseTranslator {
 
     int tokenStartIndex = node.getTokenStartIndex();
     int tokenStopIndex = node.getTokenStopIndex();
-
     Translation translation = new Translation();
     translation.tokenStopIndex = tokenStopIndex;
     translation.replacementText = replacementText;
 
     // Sanity check for overlap with regions already being expanded
     assert (tokenStopIndex >= tokenStartIndex);
-    Map.Entry<Integer, Translation> existingEntry;
-    existingEntry = translations.floorEntry(tokenStartIndex);
-    boolean prefix = false;
-    if (existingEntry != null) {
-      if (existingEntry.getKey().equals(tokenStartIndex)) {
-        if (existingEntry.getValue().tokenStopIndex == tokenStopIndex) {
-          if (existingEntry.getValue().replacementText.equals(replacementText)) {
-            // exact match for existing mapping: somebody is doing something
-            // redundant, but we'll let it pass
-            return;
-          }
-        } else if (tokenStopIndex > existingEntry.getValue().tokenStopIndex) {
-          // is existing mapping a prefix for new mapping? if so, that's also
-          // redundant, but in this case we need to expand it
-          prefix = replacementText.startsWith(
-            existingEntry.getValue().replacementText);
-          assert(prefix);
-        } else {
-          // new mapping is a prefix for existing mapping:  ignore it
-          prefix = existingEntry.getValue().replacementText.startsWith(
-            replacementText);
-          assert(prefix);
-          return;
-        }
-      }
-      if (!prefix) {
-        assert (existingEntry.getValue().tokenStopIndex < tokenStartIndex);
-      }
-    }
-    if (!prefix) {
-      existingEntry = translations.ceilingEntry(tokenStartIndex);
-      if (existingEntry != null) {
-        assert (existingEntry.getKey() > tokenStopIndex);
-      }
-    }
 
-    // Is existing entry a suffix of the newer entry and a subset of it?
-    existingEntry = translations.floorEntry(tokenStopIndex);
-    if (existingEntry != null) {
-      if (existingEntry.getKey().equals(tokenStopIndex)) {
-        if (tokenStartIndex < existingEntry.getKey() &&
-            tokenStopIndex == existingEntry.getKey()) {
-          // Seems newer entry is a super-set of existing entry, remove existing entry
-          assert (replacementText.endsWith(existingEntry.getValue().replacementText));
-          translations.remove(tokenStopIndex);
-        }
+    List<Integer> subsetEntries = new ArrayList<Integer>();
+    // Is the existing entry and newer entry are subset of one another ?
+    for (Map.Entry<Integer, Translation> existingEntry :
+          translations.headMap(tokenStopIndex, true).entrySet()) {
+      // check if the new entry contains the existing
+      if (existingEntry.getValue().tokenStopIndex <= tokenStopIndex &&
+            existingEntry.getKey() >= tokenStartIndex) {
+        // Collect newer entry is if a super-set of existing entry,
+        assert (replacementText.contains(existingEntry.getValue().replacementText));
+        subsetEntries.add(existingEntry.getKey());
+        // check if the existing entry contains the new
+      } else if (existingEntry.getValue().tokenStopIndex >= tokenStopIndex &&
+            existingEntry.getKey() <= tokenStartIndex) {
+        assert (existingEntry.getValue().replacementText.contains(replacementText));
+        // we don't need to add this new entry since there's already an overlapping one
+        return;
       }
     }
+    // remove any existing entries that are contained by the new one
+    for (Integer index : subsetEntries) {
+      translations.remove(index);
+    }
 
     // It's all good: create a new entry in the map (or update existing one)
     translations.put(tokenStartIndex, translation);

Added: hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q?rev=1506394&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q Wed Jul 24 04:02:00
2013
@@ -0,0 +1,13 @@
+drop view if exists v;
+drop view if exists w;
+
+create view v as select cast(key as string) from src;
+describe formatted v;
+
+create view w as select key, value from (
+  select key, value from src
+) a;
+describe formatted w;
+
+drop view v;
+drop view w;

Added: hive/trunk/ql/src/test/queries/clientpositive/view_cast.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/view_cast.q?rev=1506394&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/view_cast.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/view_cast.q Wed Jul 24 04:02:00 2013
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS atab; 
+CREATE TABLE IF NOT EXISTS atab (ks_uid BIGINT, sr_uid STRING, sr_id STRING, tstamp STRING,
m_id STRING, act STRING, at_sr_uid STRING, tstamp_type STRING, original_m_id STRING, original_tstamp
STRING, registered_flag TINYINT, at_ks_uid BIGINT) PARTITIONED BY (dt STRING,nt STRING); 
+LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130312', nt='tw');
+LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130311', nt='tw');
+
+DROP TABLE IF EXISTS  mstab;
+CREATE TABLE  mstab(ks_uid INT, csc INT) PARTITIONED BY (dt STRING);
+LOAD DATA LOCAL INPATH '../data/files/v2.txt' INTO TABLE mstab PARTITION (dt='20130311');
+
+DROP VIEW IF EXISTS aa_view_tw;
+CREATE VIEW aa_view_tw AS SELECT ks_uid, sr_id, act, at_ks_uid, at_sr_uid, from_unixtime(CAST(CAST(
tstamp as BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS act_date, from_unixtime(CAST(CAST( original_tstamp
AS BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS content_creation_date FROM atab WHERE dt='20130312'
AND nt='tw' AND ks_uid != at_ks_uid;
+
+DROP VIEW IF EXISTS joined_aa_view_tw;
+CREATE VIEW joined_aa_view_tw AS SELECT aa.ks_uid, aa.sr_id, aa.act, at_sr_uid, aa.act_date,
aa.at_ks_uid, aa.content_creation_date, coalesce( other.ksc, 10.0) AS at_ksc, coalesce( self.ksc
, 10.0 ) AS self_ksc FROM aa_view_tw aa LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab
WHERE dt='20130311' ) self ON ( CAST(aa.ks_uid AS BIGINT) = CAST(self.ks_uid AS BIGINT) )
LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' ) other ON ( CAST(aa.at_ks_uid
AS BIGINT) = CAST(other.ks_uid AS BIGINT) );
+
+SELECT * FROM joined_aa_view_tw;

Added: hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out?rev=1506394&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out Wed Jul 24 04:02:00
2013
@@ -0,0 +1,101 @@
+PREHOOK: query: drop view if exists v
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: drop view if exists v
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: drop view if exists w
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: drop view if exists w
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: create view v as select cast(key as string) from src
+PREHOOK: type: CREATEVIEW
+POSTHOOK: query: create view v as select cast(key as string) from src
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Output: default@v
+PREHOOK: query: describe formatted v
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted v
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+Table Type:         	VIRTUAL_VIEW        	 
+Table Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	null                	 
+InputFormat:        	org.apache.hadoop.mapred.SequenceFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+	 	 
+# View Information	 	 
+View Original Text: 	select cast(key as string) from src	 
+View Expanded Text: 	select `src`.`key` from `default`.`src`	 
+PREHOOK: query: create view w as select key, value from (
+  select key, value from src
+) a
+PREHOOK: type: CREATEVIEW
+POSTHOOK: query: create view w as select key, value from (
+  select key, value from src
+) a
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Output: default@w
+PREHOOK: query: describe formatted w
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe formatted w
+POSTHOOK: type: DESCTABLE
+# col_name            	data_type           	comment             
+	 	 
+key                 	string              	None                
+value               	string              	None                
+	 	 
+# Detailed Table Information	 	 
+Database:           	default             	 
+#### A masked pattern was here ####
+Protect Mode:       	None                	 
+Retention:          	0                   	 
+Table Type:         	VIRTUAL_VIEW        	 
+Table Parameters:	 	 
+#### A masked pattern was here ####
+	 	 
+# Storage Information	 	 
+SerDe Library:      	null                	 
+InputFormat:        	org.apache.hadoop.mapred.SequenceFileInputFormat	 
+OutputFormat:       	org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat	 
+Compressed:         	No                  	 
+Num Buckets:        	-1                  	 
+Bucket Columns:     	[]                  	 
+Sort Columns:       	[]                  	 
+	 	 
+# View Information	 	 
+View Original Text: 	select key, value from (	 
+  select key, value from src	 	 
+) a	 	 
+View Expanded Text: 	select key, value from (	 
+  select `src`.`key`, `src`.`value` from `default`.`src`	 	 
+) `a`	 	 
+PREHOOK: query: drop view v
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@v
+PREHOOK: Output: default@v
+POSTHOOK: query: drop view v
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@v
+POSTHOOK: Output: default@v
+PREHOOK: query: drop view w
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@w
+PREHOOK: Output: default@w
+POSTHOOK: query: drop view w
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@w
+POSTHOOK: Output: default@w

Added: hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out?rev=1506394&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out Wed Jul 24 04:02:00 2013
@@ -0,0 +1,85 @@
+PREHOOK: query: DROP TABLE IF EXISTS atab
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS atab
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE IF NOT EXISTS atab (ks_uid BIGINT, sr_uid STRING, sr_id STRING,
tstamp STRING, m_id STRING, act STRING, at_sr_uid STRING, tstamp_type STRING, original_m_id
STRING, original_tstamp STRING, registered_flag TINYINT, at_ks_uid BIGINT) PARTITIONED BY
(dt STRING,nt STRING)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE IF NOT EXISTS atab (ks_uid BIGINT, sr_uid STRING, sr_id STRING,
tstamp STRING, m_id STRING, act STRING, at_sr_uid STRING, tstamp_type STRING, original_m_id
STRING, original_tstamp STRING, registered_flag TINYINT, at_ks_uid BIGINT) PARTITIONED BY
(dt STRING,nt STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@atab
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130312',
nt='tw')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@atab
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION
(dt='20130312', nt='tw')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@atab
+POSTHOOK: Output: default@atab@dt=20130312/nt=tw
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130311',
nt='tw')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@atab
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION
(dt='20130311', nt='tw')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@atab
+POSTHOOK: Output: default@atab@dt=20130311/nt=tw
+PREHOOK: query: DROP TABLE IF EXISTS  mstab
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS  mstab
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE  mstab(ks_uid INT, csc INT) PARTITIONED BY (dt STRING)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE  mstab(ks_uid INT, csc INT) PARTITIONED BY (dt STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@mstab
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v2.txt' INTO TABLE mstab PARTITION
(dt='20130311')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@mstab
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v2.txt' INTO TABLE mstab PARTITION
(dt='20130311')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@mstab
+POSTHOOK: Output: default@mstab@dt=20130311
+PREHOOK: query: DROP VIEW IF EXISTS aa_view_tw
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW IF EXISTS aa_view_tw
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: CREATE VIEW aa_view_tw AS SELECT ks_uid, sr_id, act, at_ks_uid, at_sr_uid,
from_unixtime(CAST(CAST( tstamp as BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS act_date, from_unixtime(CAST(CAST(
original_tstamp AS BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS content_creation_date FROM atab
WHERE dt='20130312' AND nt='tw' AND ks_uid != at_ks_uid
+PREHOOK: type: CREATEVIEW
+POSTHOOK: query: CREATE VIEW aa_view_tw AS SELECT ks_uid, sr_id, act, at_ks_uid, at_sr_uid,
from_unixtime(CAST(CAST( tstamp as BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS act_date, from_unixtime(CAST(CAST(
original_tstamp AS BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS content_creation_date FROM atab
WHERE dt='20130312' AND nt='tw' AND ks_uid != at_ks_uid
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Output: default@aa_view_tw
+PREHOOK: query: DROP VIEW IF EXISTS joined_aa_view_tw
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW IF EXISTS joined_aa_view_tw
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: CREATE VIEW joined_aa_view_tw AS SELECT aa.ks_uid, aa.sr_id, aa.act, at_sr_uid,
aa.act_date, aa.at_ks_uid, aa.content_creation_date, coalesce( other.ksc, 10.0) AS at_ksc,
coalesce( self.ksc , 10.0 ) AS self_ksc FROM aa_view_tw aa LEFT OUTER JOIN ( SELECT ks_uid,
csc AS ksc FROM mstab WHERE dt='20130311' ) self ON ( CAST(aa.ks_uid AS BIGINT) = CAST(self.ks_uid
AS BIGINT) ) LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' )
other ON ( CAST(aa.at_ks_uid AS BIGINT) = CAST(other.ks_uid AS BIGINT) )
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@aa_view_tw
+POSTHOOK: query: CREATE VIEW joined_aa_view_tw AS SELECT aa.ks_uid, aa.sr_id, aa.act, at_sr_uid,
aa.act_date, aa.at_ks_uid, aa.content_creation_date, coalesce( other.ksc, 10.0) AS at_ksc,
coalesce( self.ksc , 10.0 ) AS self_ksc FROM aa_view_tw aa LEFT OUTER JOIN ( SELECT ks_uid,
csc AS ksc FROM mstab WHERE dt='20130311' ) self ON ( CAST(aa.ks_uid AS BIGINT) = CAST(self.ks_uid
AS BIGINT) ) LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' )
other ON ( CAST(aa.at_ks_uid AS BIGINT) = CAST(other.ks_uid AS BIGINT) )
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@aa_view_tw
+POSTHOOK: Output: default@joined_aa_view_tw
+PREHOOK: query: SELECT * FROM joined_aa_view_tw
+PREHOOK: type: QUERY
+PREHOOK: Input: default@aa_view_tw
+PREHOOK: Input: default@atab
+PREHOOK: Input: default@atab@dt=20130312/nt=tw
+PREHOOK: Input: default@joined_aa_view_tw
+PREHOOK: Input: default@mstab
+PREHOOK: Input: default@mstab@dt=20130311
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM joined_aa_view_tw
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@aa_view_tw
+POSTHOOK: Input: default@atab
+POSTHOOK: Input: default@atab@dt=20130312/nt=tw
+POSTHOOK: Input: default@joined_aa_view_tw
+POSTHOOK: Input: default@mstab
+POSTHOOK: Input: default@mstab@dt=20130311
+#### A masked pattern was here ####
+1111	abc	fun	bar	NULL	2222	NULL	99999.0	99999.0
+1111	abc	fun	bar	NULL	2222	NULL	99999.0	99999.0
+2222	abc	fun	bar	NULL	4444	NULL	99999.0	99999.0
+2222	abc	fun	bar	NULL	4444	NULL	99999.0	99999.0
+3333	abc	fun	bar	NULL	6666	NULL	99999.0	10.0
+3333	abc	fun	bar	NULL	6666	NULL	99999.0	10.0
+3333	abc	fun	bar	NULL	6666	NULL	99999.0	10.0
+4444	abc	fun	bar	NULL	8888	NULL	99999.0	99999.0



Mime
View raw message