Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 6633A10200 for ; Wed, 24 Jul 2013 04:02:32 +0000 (UTC) Received: (qmail 72375 invoked by uid 500); 24 Jul 2013 04:02:32 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 72207 invoked by uid 500); 24 Jul 2013 04:02:24 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 71966 invoked by uid 99); 24 Jul 2013 04:02:23 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Jul 2013 04:02:23 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Jul 2013 04:02:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 01B862388980; Wed, 24 Jul 2013 04:02:01 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1506394 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/ test/results/clientpositive/ Date: Wed, 24 Jul 2013 04:02:00 -0000 To: commits@hive.apache.org From: hashutosh@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130724040201.01B862388980@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: hashutosh Date: Wed Jul 24 04:02:00 2013 New Revision: 1506394 URL: http://svn.apache.org/r1506394 Log: HIVE-4547 : A complex create view statement fails with new Antlr 3.4 (Prasad Mujumdar via Ashutosh Chauhan) Added: hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q hive/trunk/ql/src/test/queries/clientpositive/view_cast.q hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java?rev=1506394&r1=1506393&r2=1506394&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/UnparseTranslator.java Wed Jul 24 04:02:00 2013 @@ -94,61 +94,35 @@ class UnparseTranslator { int tokenStartIndex = node.getTokenStartIndex(); int tokenStopIndex = node.getTokenStopIndex(); - Translation translation = new Translation(); translation.tokenStopIndex = tokenStopIndex; translation.replacementText = replacementText; // Sanity check for overlap with regions already being expanded assert (tokenStopIndex >= tokenStartIndex); - Map.Entry existingEntry; - existingEntry = translations.floorEntry(tokenStartIndex); - boolean prefix = false; - if (existingEntry != null) { - if (existingEntry.getKey().equals(tokenStartIndex)) { - if (existingEntry.getValue().tokenStopIndex == tokenStopIndex) { - if (existingEntry.getValue().replacementText.equals(replacementText)) { - // exact match for existing mapping: somebody is doing something - // redundant, but we'll let it pass - return; - } - } else if (tokenStopIndex > existingEntry.getValue().tokenStopIndex) { - // is existing mapping a prefix for new mapping? if so, that's also - // redundant, but in this case we need to expand it - prefix = replacementText.startsWith( - existingEntry.getValue().replacementText); - assert(prefix); - } else { - // new mapping is a prefix for existing mapping: ignore it - prefix = existingEntry.getValue().replacementText.startsWith( - replacementText); - assert(prefix); - return; - } - } - if (!prefix) { - assert (existingEntry.getValue().tokenStopIndex < tokenStartIndex); - } - } - if (!prefix) { - existingEntry = translations.ceilingEntry(tokenStartIndex); - if (existingEntry != null) { - assert (existingEntry.getKey() > tokenStopIndex); - } - } - // Is existing entry a suffix of the newer entry and a subset of it? - existingEntry = translations.floorEntry(tokenStopIndex); - if (existingEntry != null) { - if (existingEntry.getKey().equals(tokenStopIndex)) { - if (tokenStartIndex < existingEntry.getKey() && - tokenStopIndex == existingEntry.getKey()) { - // Seems newer entry is a super-set of existing entry, remove existing entry - assert (replacementText.endsWith(existingEntry.getValue().replacementText)); - translations.remove(tokenStopIndex); - } + List subsetEntries = new ArrayList(); + // Is the existing entry and newer entry are subset of one another ? + for (Map.Entry existingEntry : + translations.headMap(tokenStopIndex, true).entrySet()) { + // check if the new entry contains the existing + if (existingEntry.getValue().tokenStopIndex <= tokenStopIndex && + existingEntry.getKey() >= tokenStartIndex) { + // Collect newer entry is if a super-set of existing entry, + assert (replacementText.contains(existingEntry.getValue().replacementText)); + subsetEntries.add(existingEntry.getKey()); + // check if the existing entry contains the new + } else if (existingEntry.getValue().tokenStopIndex >= tokenStopIndex && + existingEntry.getKey() <= tokenStartIndex) { + assert (existingEntry.getValue().replacementText.contains(replacementText)); + // we don't need to add this new entry since there's already an overlapping one + return; } } + // remove any existing entries that are contained by the new one + for (Integer index : subsetEntries) { + translations.remove(index); + } // It's all good: create a new entry in the map (or update existing one) translations.put(tokenStartIndex, translation); Added: hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q?rev=1506394&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/create_view_translate.q Wed Jul 24 04:02:00 2013 @@ -0,0 +1,13 @@ +drop view if exists v; +drop view if exists w; + +create view v as select cast(key as string) from src; +describe formatted v; + +create view w as select key, value from ( + select key, value from src +) a; +describe formatted w; + +drop view v; +drop view w; Added: hive/trunk/ql/src/test/queries/clientpositive/view_cast.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/view_cast.q?rev=1506394&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/view_cast.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/view_cast.q Wed Jul 24 04:02:00 2013 @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS atab; +CREATE TABLE IF NOT EXISTS atab (ks_uid BIGINT, sr_uid STRING, sr_id STRING, tstamp STRING, m_id STRING, act STRING, at_sr_uid STRING, tstamp_type STRING, original_m_id STRING, original_tstamp STRING, registered_flag TINYINT, at_ks_uid BIGINT) PARTITIONED BY (dt STRING,nt STRING); +LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130312', nt='tw'); +LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130311', nt='tw'); + +DROP TABLE IF EXISTS mstab; +CREATE TABLE mstab(ks_uid INT, csc INT) PARTITIONED BY (dt STRING); +LOAD DATA LOCAL INPATH '../data/files/v2.txt' INTO TABLE mstab PARTITION (dt='20130311'); + +DROP VIEW IF EXISTS aa_view_tw; +CREATE VIEW aa_view_tw AS SELECT ks_uid, sr_id, act, at_ks_uid, at_sr_uid, from_unixtime(CAST(CAST( tstamp as BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS act_date, from_unixtime(CAST(CAST( original_tstamp AS BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS content_creation_date FROM atab WHERE dt='20130312' AND nt='tw' AND ks_uid != at_ks_uid; + +DROP VIEW IF EXISTS joined_aa_view_tw; +CREATE VIEW joined_aa_view_tw AS SELECT aa.ks_uid, aa.sr_id, aa.act, at_sr_uid, aa.act_date, aa.at_ks_uid, aa.content_creation_date, coalesce( other.ksc, 10.0) AS at_ksc, coalesce( self.ksc , 10.0 ) AS self_ksc FROM aa_view_tw aa LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' ) self ON ( CAST(aa.ks_uid AS BIGINT) = CAST(self.ks_uid AS BIGINT) ) LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' ) other ON ( CAST(aa.at_ks_uid AS BIGINT) = CAST(other.ks_uid AS BIGINT) ); + +SELECT * FROM joined_aa_view_tw; Added: hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out?rev=1506394&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/create_view_translate.q.out Wed Jul 24 04:02:00 2013 @@ -0,0 +1,101 @@ +PREHOOK: query: drop view if exists v +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view if exists v +POSTHOOK: type: DROPVIEW +PREHOOK: query: drop view if exists w +PREHOOK: type: DROPVIEW +POSTHOOK: query: drop view if exists w +POSTHOOK: type: DROPVIEW +PREHOOK: query: create view v as select cast(key as string) from src +PREHOOK: type: CREATEVIEW +POSTHOOK: query: create view v as select cast(key as string) from src +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@v +PREHOOK: query: describe formatted v +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted v +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select cast(key as string) from src +View Expanded Text: select `src`.`key` from `default`.`src` +PREHOOK: query: create view w as select key, value from ( + select key, value from src +) a +PREHOOK: type: CREATEVIEW +POSTHOOK: query: create view w as select key, value from ( + select key, value from src +) a +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@w +PREHOOK: query: describe formatted w +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe formatted w +POSTHOOK: type: DESCTABLE +# col_name data_type comment + +key string None +value string None + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Protect Mode: None +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +View Original Text: select key, value from ( + select key, value from src +) a +View Expanded Text: select key, value from ( + select `src`.`key`, `src`.`value` from `default`.`src` +) `a` +PREHOOK: query: drop view v +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@v +PREHOOK: Output: default@v +POSTHOOK: query: drop view v +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@v +POSTHOOK: Output: default@v +PREHOOK: query: drop view w +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@w +PREHOOK: Output: default@w +POSTHOOK: query: drop view w +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@w +POSTHOOK: Output: default@w Added: hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out?rev=1506394&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/view_cast.q.out Wed Jul 24 04:02:00 2013 @@ -0,0 +1,85 @@ +PREHOOK: query: DROP TABLE IF EXISTS atab +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS atab +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE IF NOT EXISTS atab (ks_uid BIGINT, sr_uid STRING, sr_id STRING, tstamp STRING, m_id STRING, act STRING, at_sr_uid STRING, tstamp_type STRING, original_m_id STRING, original_tstamp STRING, registered_flag TINYINT, at_ks_uid BIGINT) PARTITIONED BY (dt STRING,nt STRING) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE IF NOT EXISTS atab (ks_uid BIGINT, sr_uid STRING, sr_id STRING, tstamp STRING, m_id STRING, act STRING, at_sr_uid STRING, tstamp_type STRING, original_m_id STRING, original_tstamp STRING, registered_flag TINYINT, at_ks_uid BIGINT) PARTITIONED BY (dt STRING,nt STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@atab +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130312', nt='tw') +PREHOOK: type: LOAD +PREHOOK: Output: default@atab +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130312', nt='tw') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@atab +POSTHOOK: Output: default@atab@dt=20130312/nt=tw +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130311', nt='tw') +PREHOOK: type: LOAD +PREHOOK: Output: default@atab +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v1.txt' INTO TABLE atab PARTITION (dt='20130311', nt='tw') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@atab +POSTHOOK: Output: default@atab@dt=20130311/nt=tw +PREHOOK: query: DROP TABLE IF EXISTS mstab +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS mstab +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE mstab(ks_uid INT, csc INT) PARTITIONED BY (dt STRING) +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE mstab(ks_uid INT, csc INT) PARTITIONED BY (dt STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@mstab +PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v2.txt' INTO TABLE mstab PARTITION (dt='20130311') +PREHOOK: type: LOAD +PREHOOK: Output: default@mstab +POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/v2.txt' INTO TABLE mstab PARTITION (dt='20130311') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@mstab +POSTHOOK: Output: default@mstab@dt=20130311 +PREHOOK: query: DROP VIEW IF EXISTS aa_view_tw +PREHOOK: type: DROPVIEW +POSTHOOK: query: DROP VIEW IF EXISTS aa_view_tw +POSTHOOK: type: DROPVIEW +PREHOOK: query: CREATE VIEW aa_view_tw AS SELECT ks_uid, sr_id, act, at_ks_uid, at_sr_uid, from_unixtime(CAST(CAST( tstamp as BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS act_date, from_unixtime(CAST(CAST( original_tstamp AS BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS content_creation_date FROM atab WHERE dt='20130312' AND nt='tw' AND ks_uid != at_ks_uid +PREHOOK: type: CREATEVIEW +POSTHOOK: query: CREATE VIEW aa_view_tw AS SELECT ks_uid, sr_id, act, at_ks_uid, at_sr_uid, from_unixtime(CAST(CAST( tstamp as BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS act_date, from_unixtime(CAST(CAST( original_tstamp AS BIGINT)/1000 AS BIGINT),'yyyyMMdd') AS content_creation_date FROM atab WHERE dt='20130312' AND nt='tw' AND ks_uid != at_ks_uid +POSTHOOK: type: CREATEVIEW +POSTHOOK: Output: default@aa_view_tw +PREHOOK: query: DROP VIEW IF EXISTS joined_aa_view_tw +PREHOOK: type: DROPVIEW +POSTHOOK: query: DROP VIEW IF EXISTS joined_aa_view_tw +POSTHOOK: type: DROPVIEW +PREHOOK: query: CREATE VIEW joined_aa_view_tw AS SELECT aa.ks_uid, aa.sr_id, aa.act, at_sr_uid, aa.act_date, aa.at_ks_uid, aa.content_creation_date, coalesce( other.ksc, 10.0) AS at_ksc, coalesce( self.ksc , 10.0 ) AS self_ksc FROM aa_view_tw aa LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' ) self ON ( CAST(aa.ks_uid AS BIGINT) = CAST(self.ks_uid AS BIGINT) ) LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' ) other ON ( CAST(aa.at_ks_uid AS BIGINT) = CAST(other.ks_uid AS BIGINT) ) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@aa_view_tw +POSTHOOK: query: CREATE VIEW joined_aa_view_tw AS SELECT aa.ks_uid, aa.sr_id, aa.act, at_sr_uid, aa.act_date, aa.at_ks_uid, aa.content_creation_date, coalesce( other.ksc, 10.0) AS at_ksc, coalesce( self.ksc , 10.0 ) AS self_ksc FROM aa_view_tw aa LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' ) self ON ( CAST(aa.ks_uid AS BIGINT) = CAST(self.ks_uid AS BIGINT) ) LEFT OUTER JOIN ( SELECT ks_uid, csc AS ksc FROM mstab WHERE dt='20130311' ) other ON ( CAST(aa.at_ks_uid AS BIGINT) = CAST(other.ks_uid AS BIGINT) ) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@aa_view_tw +POSTHOOK: Output: default@joined_aa_view_tw +PREHOOK: query: SELECT * FROM joined_aa_view_tw +PREHOOK: type: QUERY +PREHOOK: Input: default@aa_view_tw +PREHOOK: Input: default@atab +PREHOOK: Input: default@atab@dt=20130312/nt=tw +PREHOOK: Input: default@joined_aa_view_tw +PREHOOK: Input: default@mstab +PREHOOK: Input: default@mstab@dt=20130311 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM joined_aa_view_tw +POSTHOOK: type: QUERY +POSTHOOK: Input: default@aa_view_tw +POSTHOOK: Input: default@atab +POSTHOOK: Input: default@atab@dt=20130312/nt=tw +POSTHOOK: Input: default@joined_aa_view_tw +POSTHOOK: Input: default@mstab +POSTHOOK: Input: default@mstab@dt=20130311 +#### A masked pattern was here #### +1111 abc fun bar NULL 2222 NULL 99999.0 99999.0 +1111 abc fun bar NULL 2222 NULL 99999.0 99999.0 +2222 abc fun bar NULL 4444 NULL 99999.0 99999.0 +2222 abc fun bar NULL 4444 NULL 99999.0 99999.0 +3333 abc fun bar NULL 6666 NULL 99999.0 10.0 +3333 abc fun bar NULL 6666 NULL 99999.0 10.0 +3333 abc fun bar NULL 6666 NULL 99999.0 10.0 +4444 abc fun bar NULL 8888 NULL 99999.0 99999.0