Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E04F3115B5 for ; Tue, 15 Jul 2014 18:52:12 +0000 (UTC) Received: (qmail 31385 invoked by uid 500); 15 Jul 2014 18:52:12 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 31343 invoked by uid 500); 15 Jul 2014 18:52:12 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 31332 invoked by uid 99); 15 Jul 2014 18:52:12 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Jul 2014 18:52:12 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 15 Jul 2014 18:51:55 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7EE542388C7E; Tue, 15 Jul 2014 18:51:03 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1610806 [12/13] - in /hive/branches/cbo: ./ bin/ bin/ext/ common/ common/src/java/org/apache/hadoop/hive/ant/ common/src/java/org/apache/hadoop/hive/conf/ common/src/test/org/apache/hadoop/hive/conf/ conf/ data/files/ hcatalog/core/src/mai... Date: Tue, 15 Jul 2014 18:50:55 -0000 To: commits@hive.apache.org From: hashutosh@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140715185103.7EE542388C7E@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Jul 15 18:50:51 2014 @@ -57,7 +57,6 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ArchiveUtils; -import org.apache.hadoop.hive.ql.exec.DDLTask; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.Task; @@ -117,6 +116,7 @@ import org.apache.hadoop.hive.ql.plan.Re import org.apache.hadoop.hive.ql.plan.RoleDDLDesc; import org.apache.hadoop.hive.ql.plan.ShowColumnsDesc; import org.apache.hadoop.hive.ql.plan.ShowCompactionsDesc; +import org.apache.hadoop.hive.ql.plan.ShowConfDesc; import org.apache.hadoop.hive.ql.plan.ShowCreateTableDesc; import org.apache.hadoop.hive.ql.plan.ShowDatabasesDesc; import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc; @@ -334,6 +334,10 @@ public class DDLSemanticAnalyzer extends ctx.setResFile(ctx.getLocalTmpPath()); analyzeShowTxns(ast); break; + case HiveParser.TOK_SHOWCONF: + ctx.setResFile(ctx.getLocalTmpPath()); + analyzeShowConf(ast); + break; case HiveParser.TOK_DESCFUNCTION: ctx.setResFile(ctx.getLocalTmpPath()); analyzeDescFunction(ast); @@ -1032,17 +1036,16 @@ public class DDLSemanticAnalyzer extends String indexComment = null; RowFormatParams rowFormatParams = new RowFormatParams(); - StorageFormat storageFormat = new StorageFormat(); - AnalyzeCreateCommonVars shared = new AnalyzeCreateCommonVars(); + StorageFormat storageFormat = new StorageFormat(conf); for (int idx = 4; idx < ast.getChildCount(); idx++) { ASTNode child = (ASTNode) ast.getChild(idx); - if (storageFormat.fillStorageFormat(child, shared)) { + if (storageFormat.fillStorageFormat(child)) { continue; } switch (child.getToken().getType()) { case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(shared, child); + rowFormatParams.analyzeRowFormat(child); break; case HiveParser.TOK_CREATEINDEX_INDEXTBLNAME: ASTNode ch = (ASTNode) child.getChild(0); @@ -1063,10 +1066,10 @@ public class DDLSemanticAnalyzer extends break; case HiveParser.TOK_TABLESERIALIZER: child = (ASTNode) child.getChild(0); - shared.serde = unescapeSQLString(child.getChild(0).getText()); + storageFormat.setSerde(unescapeSQLString(child.getChild(0).getText())); if (child.getChildCount() == 2) { readProps((ASTNode) (child.getChild(1).getChild(0)), - shared.serdeProps); + storageFormat.getSerdeProps()); } break; case HiveParser.TOK_INDEXCOMMENT: @@ -1075,14 +1078,14 @@ public class DDLSemanticAnalyzer extends } } - storageFormat.fillDefaultStorageFormat(shared); + storageFormat.fillDefaultStorageFormat(); CreateIndexDesc crtIndexDesc = new CreateIndexDesc(tableName, indexName, - indexedCols, indexTableName, deferredRebuild, storageFormat.inputFormat, - storageFormat.outputFormat, - storageFormat.storageHandler, typeName, location, idxProps, tblProps, - shared.serde, shared.serdeProps, rowFormatParams.collItemDelim, + indexedCols, indexTableName, deferredRebuild, storageFormat.getInputFormat(), + storageFormat.getOutputFormat(), + storageFormat.getStorageHandler(), typeName, location, idxProps, tblProps, + storageFormat.getSerde(), storageFormat.getSerdeProps(), rowFormatParams.collItemDelim, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.lineDelim, rowFormatParams.mapKeyDelim, indexComment); Task createIndex = @@ -1317,69 +1320,15 @@ public class DDLSemanticAnalyzer extends HashMap partSpec) throws SemanticException { - String inputFormat = null; - String outputFormat = null; - String storageHandler = null; - String serde = null; + StorageFormat format = new StorageFormat(conf); ASTNode child = (ASTNode) ast.getChild(0); - switch (child.getToken().getType()) { - case HiveParser.TOK_TABLEFILEFORMAT: - inputFormat = unescapeSQLString(((ASTNode) child.getChild(0)).getToken() - .getText()); - outputFormat = unescapeSQLString(((ASTNode) child.getChild(1)).getToken() - .getText()); - serde = unescapeSQLString(((ASTNode) child.getChild(2)).getToken() - .getText()); - try { - Class.forName(inputFormat); - Class.forName(outputFormat); - Class.forName(serde); - } catch (ClassNotFoundException e) { - throw new SemanticException(e); - } - break; - case HiveParser.TOK_STORAGEHANDLER: - storageHandler = - unescapeSQLString(((ASTNode) child.getChild(1)).getToken().getText()); - try { - Class.forName(storageHandler); - } catch (ClassNotFoundException e) { - throw new SemanticException(e); - } - break; - case HiveParser.TOK_TBLSEQUENCEFILE: - inputFormat = SEQUENCEFILE_INPUT; - outputFormat = SEQUENCEFILE_OUTPUT; - serde = org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName(); - break; - case HiveParser.TOK_TBLTEXTFILE: - inputFormat = TEXTFILE_INPUT; - outputFormat = TEXTFILE_OUTPUT; - serde = org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName(); - break; - case HiveParser.TOK_TBLRCFILE: - inputFormat = RCFILE_INPUT; - outputFormat = RCFILE_OUTPUT; - serde = conf.getVar(HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE); - break; - case HiveParser.TOK_TBLORCFILE: - inputFormat = ORCFILE_INPUT; - outputFormat = ORCFILE_OUTPUT; - serde = ORCFILE_SERDE; - break; - case HiveParser.TOK_TBLPARQUETFILE: - inputFormat = PARQUETFILE_INPUT; - outputFormat = PARQUETFILE_OUTPUT; - serde = PARQUETFILE_SERDE; - break; - case HiveParser.TOK_FILEFORMAT_GENERIC: - handleGenericFileFormat(child); - break; + if (!format.fillStorageFormat(child)) { + throw new AssertionError("Unknown token " + child.getText()); } - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, inputFormat, - outputFormat, serde, storageHandler, partSpec); + AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, format.getInputFormat(), + format.getOutputFormat(), format.getSerde(), format.getStorageHandler(), partSpec); addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), @@ -2338,7 +2287,15 @@ public class DDLSemanticAnalyzer extends ctx.setNeedLockMgr(true); } - /** + private void analyzeShowConf(ASTNode ast) throws SemanticException { + String confName = stripQuotes(ast.getChild(0).getText()); + ShowConfDesc showConfDesc = new ShowConfDesc(ctx.getResFile(), confName); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), + showConfDesc), conf)); + setFetchTask(createFetchTask(showConfDesc.getSchema())); + } + + /** * Add the task according to the parsed command tree. This is used for the CLI * command "LOCK TABLE ..;". * Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Jul 15 18:50:51 2014 @@ -151,11 +151,6 @@ KW_KEY_TYPE: '$KEY$'; KW_LINES: 'LINES'; KW_STORED: 'STORED'; KW_FILEFORMAT: 'FILEFORMAT'; -KW_SEQUENCEFILE: 'SEQUENCEFILE'; -KW_TEXTFILE: 'TEXTFILE'; -KW_RCFILE: 'RCFILE'; -KW_ORCFILE: 'ORC'; -KW_PARQUETFILE: 'PARQUET'; KW_INPUTFORMAT: 'INPUTFORMAT'; KW_OUTPUTFORMAT: 'OUTPUTFORMAT'; KW_INPUTDRIVER: 'INPUTDRIVER'; @@ -296,6 +291,7 @@ KW_COMPACTIONS: 'COMPACTIONS'; KW_TRANSACTIONS: 'TRANSACTIONS'; KW_REWRITE : 'REWRITE'; KW_AUTHORIZATION: 'AUTHORIZATION'; +KW_CONF: 'CONF'; // Operators // NOTE: if you add a new function/operator, add it to sysFuncNames so that describe function _FUNC_ will work. Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Jul 15 18:50:51 2014 @@ -163,6 +163,7 @@ TOK_SHOW_CREATETABLE; TOK_SHOW_TABLESTATUS; TOK_SHOW_TBLPROPERTIES; TOK_SHOWLOCKS; +TOK_SHOWCONF; TOK_LOCKTABLE; TOK_UNLOCKTABLE; TOK_LOCKDB; @@ -182,11 +183,6 @@ TOK_TABLEROWFORMATCOLLITEMS; TOK_TABLEROWFORMATMAPKEYS; TOK_TABLEROWFORMATLINES; TOK_TABLEROWFORMATNULL; -TOK_TBLORCFILE; -TOK_TBLPARQUETFILE; -TOK_TBLSEQUENCEFILE; -TOK_TBLTEXTFILE; -TOK_TBLRCFILE; TOK_TABLEFILEFORMAT; TOK_FILEFORMAT_GENERIC; TOK_OFFLINE; @@ -267,6 +263,7 @@ TOK_ROLE; TOK_RESOURCE_ALL; TOK_GRANT_WITH_OPTION; TOK_GRANT_WITH_ADMIN_OPTION; +TOK_ADMIN_OPTION_FOR; TOK_PRIV_ALL; TOK_PRIV_ALTER_METADATA; TOK_PRIV_ALTER_DATA; @@ -1265,12 +1262,7 @@ alterStatementSuffixCompact fileFormat @init { pushMsg("file format specification", state); } @after { popMsg(state); } - : KW_SEQUENCEFILE -> ^(TOK_TBLSEQUENCEFILE) - | KW_TEXTFILE -> ^(TOK_TBLTEXTFILE) - | KW_RCFILE -> ^(TOK_TBLRCFILE) - | KW_ORCFILE -> ^(TOK_TBLORCFILE) - | KW_PARQUETFILE -> ^(TOK_TBLPARQUETFILE) - | KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral KW_SERDE serdeCls=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + : KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral KW_SERDE serdeCls=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $serdeCls $inDriver? $outDriver?) | genericSpec=identifier -> ^(TOK_FILEFORMAT_GENERIC $genericSpec) ; @@ -1336,6 +1328,7 @@ showStatement -> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?) | KW_SHOW KW_COMPACTIONS -> ^(TOK_SHOW_COMPACTIONS) | KW_SHOW KW_TRANSACTIONS -> ^(TOK_SHOW_TRANSACTIONS) + | KW_SHOW KW_CONF StringLiteral -> ^(TOK_SHOWCONF StringLiteral) ; lockStatement @@ -1409,8 +1402,8 @@ grantRole revokeRole @init {pushMsg("revoke role", state);} @after {popMsg(state);} - : KW_REVOKE KW_ROLE? identifier (COMMA identifier)* KW_FROM principalSpecification withAdminOption? - -> ^(TOK_REVOKE_ROLE principalSpecification withAdminOption? identifier+) + : KW_REVOKE adminOptionFor? KW_ROLE? identifier (COMMA identifier)* KW_FROM principalSpecification + -> ^(TOK_REVOKE_ROLE principalSpecification adminOptionFor? identifier+) ; showRoleGrants @@ -1533,6 +1526,13 @@ withGrantOption -> ^(TOK_GRANT_WITH_OPTION) ; +adminOptionFor +@init {pushMsg("admin option for", state);} +@after {popMsg(state);} + : KW_ADMIN KW_OPTION KW_FOR + -> ^(TOK_ADMIN_OPTION_FOR) +; + withAdminOption @init {pushMsg("with admin option", state);} @after {popMsg(state);} @@ -1808,12 +1808,7 @@ tableFileFormat @init { pushMsg("table file format specification", state); } @after { popMsg(state); } : - KW_STORED KW_AS KW_SEQUENCEFILE -> TOK_TBLSEQUENCEFILE - | KW_STORED KW_AS KW_TEXTFILE -> TOK_TBLTEXTFILE - | KW_STORED KW_AS KW_RCFILE -> TOK_TBLRCFILE - | KW_STORED KW_AS KW_ORCFILE -> TOK_TBLORCFILE - | KW_STORED KW_AS KW_PARQUETFILE -> TOK_TBLPARQUETFILE - | KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? + KW_STORED KW_AS KW_INPUTFORMAT inFmt=StringLiteral KW_OUTPUTFORMAT outFmt=StringLiteral (KW_INPUTDRIVER inDriver=StringLiteral KW_OUTPUTDRIVER outDriver=StringLiteral)? -> ^(TOK_TABLEFILEFORMAT $inFmt $outFmt $inDriver? $outDriver?) | KW_STORED KW_BY storageHandler=StringLiteral (KW_WITH KW_SERDEPROPERTIES serdeprops=tableProperties)? Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Jul 15 18:50:51 2014 @@ -538,5 +538,5 @@ functionIdentifier nonReserved : - KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_PARQUETFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION + KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_AN ALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN | KW_JAR | KW_FILE | KW_OWNER | KW_PRINCIPALS | KW_ALL | KW_DEFAULT | KW_NONE | KW_COMPACT | KW_COMPACTIONS | KW_TRANSACTIONS | KW_REWRITE | KW_AUTHORIZATION ; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java Tue Jul 15 18:50:51 2014 @@ -22,12 +22,11 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -216,4 +215,18 @@ public final class ParseUtils { return TypeInfoFactory.getDecimalTypeInfo(precision, scale); } + public static String ensureClassExists(String className) + throws SemanticException { + if (className == null) { + return null; + } + try { + Class.forName(className, true, JavaUtils.getClassLoader()); + } catch (ClassNotFoundException e) { + throw new SemanticException("Cannot find class '" + className + "'", e); + } + return className; + } + + } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jul 15 18:50:51 2014 @@ -1273,10 +1273,6 @@ public class SemanticAnalyzer extends Ba return phase1Result; } - private void getMetaData(QBExpr qbexpr) throws SemanticException { - getMetaData(qbexpr, null); - } - private void getMetaData(QBExpr qbexpr, ReadEntity parentInput) throws SemanticException { if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) { @@ -1456,8 +1452,7 @@ public class SemanticAnalyzer extends Ba } RowFormatParams rowFormatParams = new RowFormatParams(); - AnalyzeCreateCommonVars shared = new AnalyzeCreateCommonVars(); - StorageFormat storageFormat = new StorageFormat(); + StorageFormat storageFormat = new StorageFormat(conf); LOG.info("Get metadata for destination tables"); // Go over all the destination structures and populate the related @@ -1552,10 +1547,16 @@ public class SemanticAnalyzer extends Ba int numCh = ast.getChildCount(); for (int num = 1; num < numCh ; num++){ ASTNode child = (ASTNode) ast.getChild(num); - if (ast.getChild(num) != null){ + if (child != null) { + if (storageFormat.fillStorageFormat(child)) { + localDirectoryDesc.setOutputFormat(storageFormat.getOutputFormat()); + localDirectoryDesc.setSerName(storageFormat.getSerde()); + localDirectoryDescIsSet = true; + continue; + } switch (child.getToken().getType()) { case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(shared, child); + rowFormatParams.analyzeRowFormat(child); localDirectoryDesc.setFieldDelim(rowFormatParams.fieldDelim); localDirectoryDesc.setLineDelim(rowFormatParams.lineDelim); localDirectoryDesc.setCollItemDelim(rowFormatParams.collItemDelim); @@ -1566,18 +1567,8 @@ public class SemanticAnalyzer extends Ba break; case HiveParser.TOK_TABLESERIALIZER: ASTNode serdeChild = (ASTNode) child.getChild(0); - shared.serde = unescapeSQLString(serdeChild.getChild(0).getText()); - localDirectoryDesc.setSerName(shared.serde); - localDirectoryDescIsSet=true; - break; - case HiveParser.TOK_TBLSEQUENCEFILE: - case HiveParser.TOK_TBLTEXTFILE: - case HiveParser.TOK_TBLRCFILE: - case HiveParser.TOK_TBLORCFILE: - case HiveParser.TOK_TABLEFILEFORMAT: - storageFormat.fillStorageFormat(child, shared); - localDirectoryDesc.setOutputFormat(storageFormat.outputFormat); - localDirectoryDesc.setSerName(shared.serde); + storageFormat.setSerde(unescapeSQLString(serdeChild.getChild(0).getText())); + localDirectoryDesc.setSerName(storageFormat.getSerde()); localDirectoryDescIsSet=true; break; } @@ -10186,8 +10177,7 @@ public class SemanticAnalyzer extends Ba boolean storedAsDirs = false; RowFormatParams rowFormatParams = new RowFormatParams(); - StorageFormat storageFormat = new StorageFormat(); - AnalyzeCreateCommonVars shared = new AnalyzeCreateCommonVars(); + StorageFormat storageFormat = new StorageFormat(conf); LOG.info("Creating table " + tableName + " position=" + ast.getCharPositionInLine()); @@ -10201,7 +10191,7 @@ public class SemanticAnalyzer extends Ba */ for (int num = 1; num < numCh; num++) { ASTNode child = (ASTNode) ast.getChild(num); - if (storageFormat.fillStorageFormat(child, shared)) { + if (storageFormat.fillStorageFormat(child)) { continue; } switch (child.getToken().getType()) { @@ -10273,7 +10263,7 @@ public class SemanticAnalyzer extends Ba } break; case HiveParser.TOK_TABLEROWFORMAT: - rowFormatParams.analyzeRowFormat(shared, child); + rowFormatParams.analyzeRowFormat(child); break; case HiveParser.TOK_TABLELOCATION: location = unescapeSQLString(child.getChild(0).getText()); @@ -10285,16 +10275,12 @@ public class SemanticAnalyzer extends Ba break; case HiveParser.TOK_TABLESERIALIZER: child = (ASTNode) child.getChild(0); - shared.serde = unescapeSQLString(child.getChild(0).getText()); + storageFormat.setSerde(unescapeSQLString(child.getChild(0).getText())); if (child.getChildCount() == 2) { readProps((ASTNode) (child.getChild(1).getChild(0)), - shared.serdeProps); + storageFormat.getSerdeProps()); } break; - - case HiveParser.TOK_FILEFORMAT_GENERIC: - handleGenericFileFormat(child); - break; case HiveParser.TOK_TABLESKEWED: /** * Throw an error if the user tries to use the DDL with @@ -10315,9 +10301,9 @@ public class SemanticAnalyzer extends Ba } } - storageFormat.fillDefaultStorageFormat(shared); + storageFormat.fillDefaultStorageFormat(); - if ((command_type == CTAS) && (storageFormat.storageHandler != null)) { + if ((command_type == CTAS) && (storageFormat.getStorageHandler() != null)) { throw new SemanticException(ErrorMsg.CREATE_NON_NATIVE_AS.getMsg()); } @@ -10329,7 +10315,8 @@ public class SemanticAnalyzer extends Ba return null; } } catch (HiveException e) { - e.printStackTrace(); + // should not occur since second parameter to getTableWithQN is false + throw new IllegalStateException("Unxpected Exception thrown: " + e.getMessage(), e); } } @@ -10370,8 +10357,8 @@ public class SemanticAnalyzer extends Ba rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, comment, - storageFormat.inputFormat, storageFormat.outputFormat, location, shared.serde, - storageFormat.storageHandler, shared.serdeProps, tblProps, ifNotExists, skewedColNames, + storageFormat.getInputFormat(), storageFormat.getOutputFormat(), location, storageFormat.getSerde(), + storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists, skewedColNames, skewedValues); crtTblDesc.setStoredAsSubDirectories(storedAsDirs); crtTblDesc.setNullFormat(rowFormatParams.nullFormat); @@ -10395,8 +10382,9 @@ public class SemanticAnalyzer extends Ba } } CreateTableLikeDesc crtTblLikeDesc = new CreateTableLikeDesc(tableName, isExt, isTemporary, - storageFormat.inputFormat, storageFormat.outputFormat, location, - shared.serde, shared.serdeProps, tblProps, ifNotExists, likeTableName); + storageFormat.getInputFormat(), storageFormat.getOutputFormat(), location, + storageFormat.getSerde(), storageFormat.getSerdeProps(), tblProps, ifNotExists, + likeTableName); SessionState.get().setCommandType(HiveOperation.CREATETABLE); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), crtTblLikeDesc), conf)); @@ -10420,10 +10408,10 @@ public class SemanticAnalyzer extends Ba bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim, rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim, - comment, storageFormat.inputFormat, - storageFormat.outputFormat, location, shared.serde, storageFormat.storageHandler, - shared.serdeProps, - tblProps, ifNotExists, skewedColNames, skewedValues); + comment, storageFormat.getInputFormat(), + storageFormat.getOutputFormat(), location, storageFormat.getSerde(), + storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists, + skewedColNames, skewedValues); crtTblDesc.setStoredAsSubDirectories(storedAsDirs); crtTblDesc.setNullFormat(rowFormatParams.nullFormat); qb.setTableDesc(crtTblDesc); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Tue Jul 15 18:50:51 2014 @@ -69,6 +69,7 @@ public final class SemanticAnalyzerFacto commandType.put(HiveParser.TOK_SHOWPARTITIONS, HiveOperation.SHOWPARTITIONS); commandType.put(HiveParser.TOK_SHOWLOCKS, HiveOperation.SHOWLOCKS); commandType.put(HiveParser.TOK_SHOWDBLOCKS, HiveOperation.SHOWLOCKS); + commandType.put(HiveParser.TOK_SHOWCONF, HiveOperation.SHOWCONF); commandType.put(HiveParser.TOK_CREATEFUNCTION, HiveOperation.CREATEFUNCTION); commandType.put(HiveParser.TOK_DROPFUNCTION, HiveOperation.DROPFUNCTION); commandType.put(HiveParser.TOK_CREATEMACRO, HiveOperation.CREATEMACRO); @@ -203,6 +204,7 @@ public final class SemanticAnalyzerFacto case HiveParser.TOK_SHOWDBLOCKS: case HiveParser.TOK_SHOW_COMPACTIONS: case HiveParser.TOK_SHOW_TRANSACTIONS: + case HiveParser.TOK_SHOWCONF: case HiveParser.TOK_CREATEINDEX: case HiveParser.TOK_DROPINDEX: case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/VariableSubstitution.java Tue Jul 15 18:50:51 2014 @@ -17,79 +17,44 @@ */ package org.apache.hadoop.hive.ql.parse; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.processors.SetProcessor; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.conf.SystemVariables; + +import java.util.Map; -public class VariableSubstitution { +public class VariableSubstitution extends SystemVariables { private static final Log l4j = LogFactory.getLog(VariableSubstitution.class); - protected static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}"); - private String getSubstitute(HiveConf conf, String var) { - String val = null; - try { - if (var.startsWith(SetProcessor.SYSTEM_PREFIX)) { - val = System.getProperty(var.substring(SetProcessor.SYSTEM_PREFIX.length())); - } - } catch(SecurityException se) { - l4j.warn("Unexpected SecurityException in Configuration", se); - } - if (val ==null){ - if (var.startsWith(SetProcessor.ENV_PREFIX)){ - val = System.getenv(var.substring(SetProcessor.ENV_PREFIX.length())); - } - } - if (val == null) { - if (var.startsWith(SetProcessor.HIVECONF_PREFIX)){ - val = conf.get(var.substring(SetProcessor.HIVECONF_PREFIX.length())); - } - } - if (val ==null){ - if(var.startsWith(SetProcessor.HIVEVAR_PREFIX)){ - val = SessionState.get().getHiveVariables().get(var.substring(SetProcessor.HIVEVAR_PREFIX.length())); + @Override + protected String getSubstitute(Configuration conf, String var) { + String val = super.getSubstitute(conf, var); + if (val == null && SessionState.get() != null) { + Map vars = SessionState.get().getHiveVariables(); + if (var.startsWith(HIVEVAR_PREFIX)) { + val = vars.get(var.substring(HIVEVAR_PREFIX.length())); } else { - val = SessionState.get().getHiveVariables().get(var); + val = vars.get(var); } } return val; } - public String substitute (HiveConf conf, String expr) { - - if (conf.getBoolVar(ConfVars.HIVEVARIABLESUBSTITUTE)){ - l4j.debug("Substitution is on: "+expr); - } else { - return expr; - } + public String substitute(HiveConf conf, String expr) { if (expr == null) { - return null; + return expr; } - Matcher match = varPat.matcher(""); - String eval = expr; - for(int s=0;s inputs, HashSet outputs, + ShowConfDesc showConfDesc) { + this(inputs, outputs); + this.showConfDesc = showConfDesc; + } + public DescDatabaseDesc getDescDatabaseDesc() { return descDbDesc; } @@ -1117,4 +1125,12 @@ public class DDLWork implements Serializ AlterTableExchangePartition alterTableExchangePartition) { this.alterTableExchangePartition = alterTableExchangePartition; } + + public ShowConfDesc getShowConfDesc() { + return showConfDesc; + } + + public void setShowConfDesc(ShowConfDesc showConfDesc) { + this.showConfDesc = showConfDesc; + } } Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java Tue Jul 15 18:50:51 2014 @@ -67,6 +67,7 @@ public enum HiveOperation { SHOWINDEXES("SHOWINDEXES", null, null), SHOWPARTITIONS("SHOWPARTITIONS", null, null), SHOWLOCKS("SHOWLOCKS", null, null), + SHOWCONF("SHOWCONF", null, null), CREATEFUNCTION("CREATEFUNCTION", null, null), DROPFUNCTION("DROPFUNCTION", null, null), CREATEMACRO("CREATEMACRO", null, null), Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java Tue Jul 15 18:50:51 2014 @@ -22,6 +22,8 @@ import static org.apache.hadoop.hive.ser import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME; import static org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.defaultNullString; +import static org.apache.hadoop.hive.conf.SystemVariables.*; + import java.util.Map; import java.util.Properties; import java.util.SortedMap; @@ -39,12 +41,7 @@ import org.apache.hadoop.hive.ql.session */ public class SetProcessor implements CommandProcessor { - private static String prefix = "set: "; - public static final String ENV_PREFIX = "env:"; - public static final String SYSTEM_PREFIX = "system:"; - public static final String HIVECONF_PREFIX = "hiveconf:"; - public static final String HIVEVAR_PREFIX = "hivevar:"; - public static final String SET_COLUMN_NAME = "set"; + private static final String prefix = "set: "; public static boolean getBoolean(String value) { if (value.equals("on") || value.equals("true")) { @@ -69,7 +66,7 @@ public class SetProcessor implements Com // Inserting hive variables for (String s : ss.getHiveVariables().keySet()) { - sortedMap.put(SetProcessor.HIVEVAR_PREFIX + s, ss.getHiveVariables().get(s)); + sortedMap.put(HIVEVAR_PREFIX + s, ss.getHiveVariables().get(s)); } for (Map.Entry entries : sortedMap.entrySet()) { @@ -117,17 +114,17 @@ public class SetProcessor implements Com ss.err.println("Warning: Value had a \\n character in it."); } varname = varname.trim(); - if (varname.startsWith(SetProcessor.ENV_PREFIX)){ + if (varname.startsWith(ENV_PREFIX)){ ss.err.println("env:* variables can not be set."); return 1; - } else if (varname.startsWith(SetProcessor.SYSTEM_PREFIX)){ - String propName = varname.substring(SetProcessor.SYSTEM_PREFIX.length()); + } else if (varname.startsWith(SYSTEM_PREFIX)){ + String propName = varname.substring(SYSTEM_PREFIX.length()); System.getProperties().setProperty(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); - } else if (varname.startsWith(SetProcessor.HIVECONF_PREFIX)){ - String propName = varname.substring(SetProcessor.HIVECONF_PREFIX.length()); + } else if (varname.startsWith(HIVECONF_PREFIX)){ + String propName = varname.substring(HIVECONF_PREFIX.length()); setConf(varname, propName, varvalue, false); - } else if (varname.startsWith(SetProcessor.HIVEVAR_PREFIX)) { - String propName = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); + } else if (varname.startsWith(HIVEVAR_PREFIX)) { + String propName = varname.substring(HIVEVAR_PREFIX.length()); ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue)); } else { setConf(varname, varname, varvalue, true); @@ -169,7 +166,7 @@ public class SetProcessor implements Com private SortedMap propertiesToSortedMap(Properties p){ SortedMap sortedPropMap = new TreeMap(); - for (Map.Entry entry :System.getProperties().entrySet() ){ + for (Map.Entry entry : p.entrySet() ){ sortedPropMap.put( (String) entry.getKey(), (String) entry.getValue()); } return sortedPropMap; @@ -188,38 +185,38 @@ public class SetProcessor implements Com ss.out.println("silent" + "=" + ss.getIsSilent()); return createProcessorSuccessResponse(); } - if (varname.startsWith(SetProcessor.SYSTEM_PREFIX)){ - String propName = varname.substring(SetProcessor.SYSTEM_PREFIX.length()); + if (varname.startsWith(SYSTEM_PREFIX)) { + String propName = varname.substring(SYSTEM_PREFIX.length()); String result = System.getProperty(propName); - if (result != null){ - ss.out.println(SetProcessor.SYSTEM_PREFIX+propName + "=" + result); + if (result != null) { + ss.out.println(SYSTEM_PREFIX + propName + "=" + result); return createProcessorSuccessResponse(); } else { - ss.out.println( propName + " is undefined as a system property"); + ss.out.println(propName + " is undefined as a system property"); return new CommandProcessorResponse(1); } - } else if (varname.indexOf(SetProcessor.ENV_PREFIX)==0){ + } else if (varname.indexOf(ENV_PREFIX) == 0) { String var = varname.substring(ENV_PREFIX.length()); - if (System.getenv(var)!=null){ - ss.out.println(SetProcessor.ENV_PREFIX+var + "=" + System.getenv(var)); + if (System.getenv(var) != null) { + ss.out.println(ENV_PREFIX + var + "=" + System.getenv(var)); return createProcessorSuccessResponse(); } else { ss.out.println(varname + " is undefined as an environmental variable"); return new CommandProcessorResponse(1); } - } else if (varname.indexOf(SetProcessor.HIVECONF_PREFIX)==0) { - String var = varname.substring(SetProcessor.HIVECONF_PREFIX.length()); - if (ss.getConf().get(var)!=null){ - ss.out.println(SetProcessor.HIVECONF_PREFIX+var + "=" + ss.getConf().get(var)); + } else if (varname.indexOf(HIVECONF_PREFIX) == 0) { + String var = varname.substring(HIVECONF_PREFIX.length()); + if (ss.getConf().get(var) != null) { + ss.out.println(HIVECONF_PREFIX + var + "=" + ss.getConf().get(var)); return createProcessorSuccessResponse(); } else { ss.out.println(varname + " is undefined as a hive configuration variable"); return new CommandProcessorResponse(1); } - } else if (varname.indexOf(SetProcessor.HIVEVAR_PREFIX)==0) { - String var = varname.substring(SetProcessor.HIVEVAR_PREFIX.length()); - if (ss.getHiveVariables().get(var)!=null){ - ss.out.println(SetProcessor.HIVEVAR_PREFIX+var + "=" + ss.getHiveVariables().get(var)); + } else if (varname.indexOf(HIVEVAR_PREFIX) == 0) { + String var = varname.substring(HIVEVAR_PREFIX.length()); + if (ss.getHiveVariables().get(var) != null) { + ss.out.println(HIVEVAR_PREFIX + var + "=" + ss.getHiveVariables().get(var)); return createProcessorSuccessResponse(); } else { ss.out.println(varname + " is undefined as a hive variable"); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java Tue Jul 15 18:50:51 2014 @@ -69,6 +69,7 @@ public enum HiveOperationType { SHOWINDEXES, SHOWPARTITIONS, SHOWLOCKS, + SHOWCONF, CREATEFUNCTION, DROPFUNCTION, CREATEMACRO, Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java Tue Jul 15 18:50:51 2014 @@ -46,7 +46,7 @@ public class Operation2Privilege { private PrivRequirement(SQLPrivTypeGrant[] privs, IOType ioType) { - this(privs, ioType, (HivePrivObjectActionType) null); + this(privs, ioType, null); } private PrivRequirement(SQLPrivTypeGrant[] privs, IOType ioType, @@ -291,6 +291,8 @@ public class Operation2Privilege { (null, null)); op2Priv.put(HiveOperationType.SHOW_TRANSACTIONS, PrivRequirement.newIOPrivRequirement (null, null)); + op2Priv.put(HiveOperationType.SHOWCONF, PrivRequirement.newIOPrivRequirement +(null, null)); op2Priv.put(HiveOperationType.LOCKTABLE, PrivRequirement.newIOPrivRequirement (null, null)); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLStdHiveAccessController.java Tue Jul 15 18:50:51 2014 @@ -79,7 +79,7 @@ public class SQLStdHiveAccessController private HiveRoleGrant adminRole; private final String ADMIN_ONLY_MSG = "User has to belong to ADMIN role and " + "have it as current role, for this action."; - private final String HAS_ADMIN_PRIV_MSG = "grantor need to have ADMIN privileges on role being" + private final String HAS_ADMIN_PRIV_MSG = "grantor need to have ADMIN OPTION on role being" + " granted and have it as a current role for this action."; public static final Log LOG = LogFactory.getLog(SQLStdHiveAccessController.class); @@ -308,11 +308,6 @@ public class SQLStdHiveAccessController public void revokeRole(List hivePrincipals, List roleNames, boolean grantOption, HivePrincipal grantorPrinc) throws HiveAuthzPluginException, HiveAccessControlException { - if (grantOption) { - // removing grant privileges only is not supported in metastore api - throw new HiveAuthzPluginException("Revoking only the admin privileges on " - + "role is not currently supported"); - } if (!(isUserAdmin() || doesUserHasAdminOption(roleNames))) { throw new HiveAccessControlException("Current user : " + currentUserName+ " is not" + " allowed to revoke role. " + ADMIN_ONLY_MSG + " Otherwise, " + HAS_ADMIN_PRIV_MSG); @@ -322,7 +317,7 @@ public class SQLStdHiveAccessController try { IMetaStoreClient mClient = metastoreClientFactory.getHiveMetastoreClient(); mClient.revoke_role(roleName, hivePrincipal.getName(), - AuthorizationUtils.getThriftPrincipalType(hivePrincipal.getType())); + AuthorizationUtils.getThriftPrincipalType(hivePrincipal.getType()), grantOption); } catch (Exception e) { String msg = "Error revoking roles for " + hivePrincipal.getName() + " to role " + roleName + ": " + e.getMessage(); Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFormatNumber.java Tue Jul 15 18:50:51 2014 @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf.ge import java.text.DecimalFormat; +import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; @@ -32,6 +33,7 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.Text; @@ -78,7 +80,8 @@ public class GenericUDFFormatNumber exte + " or \"" + serdeConstants.INT_TYPE_NAME + "\"" + " or \"" + serdeConstants.BIGINT_TYPE_NAME + "\"" + " or \"" + serdeConstants.DOUBLE_TYPE_NAME + "\"" - + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\", but \"" + + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\"" + + " or \"" + serdeConstants.DECIMAL_TYPE_NAME + "\", but \"" + arguments[0].getTypeName() + "\" was found."); } @@ -104,8 +107,9 @@ public class GenericUDFFormatNumber exte case SHORT: case INT: case LONG: - case FLOAT: case DOUBLE: + case FLOAT: + case DECIMAL: break; default: throw new UDFArgumentTypeException(0, "Argument 1" @@ -115,7 +119,8 @@ public class GenericUDFFormatNumber exte + " or \"" + serdeConstants.INT_TYPE_NAME + "\"" + " or \"" + serdeConstants.BIGINT_TYPE_NAME + "\"" + " or \"" + serdeConstants.DOUBLE_TYPE_NAME + "\"" - + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\", but \"" + + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\"" + + " or \"" + serdeConstants.DECIMAL_TYPE_NAME + "\", but \"" + arguments[0].getTypeName() + "\" was found."); } @@ -168,6 +173,7 @@ public class GenericUDFFormatNumber exte double xDoubleValue = 0.0; float xFloatValue = 0.0f; + HiveDecimal xDecimalValue = null; int xIntValue = 0; long xLongValue = 0L; @@ -182,6 +188,11 @@ public class GenericUDFFormatNumber exte xFloatValue = ((FloatObjectInspector) argumentOIs[0]).get(arguments[0].get()); resultText.set(numberFormat.format(xFloatValue)); break; + case DECIMAL: + xDecimalValue = ((HiveDecimalObjectInspector) argumentOIs[0]) + .getPrimitiveJavaObject(arguments[0].get()); + resultText.set(numberFormat.format(xDecimalValue.bigDecimalValue())); + break; case BYTE: case SHORT: case INT: @@ -199,7 +210,8 @@ public class GenericUDFFormatNumber exte + " or \"" + serdeConstants.INT_TYPE_NAME + "\"" + " or \"" + serdeConstants.BIGINT_TYPE_NAME + "\"" + " or \"" + serdeConstants.DOUBLE_TYPE_NAME + "\"" - + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\", but \"" + + " or \"" + serdeConstants.FLOAT_TYPE_NAME + "\"" + + " or \"" + serdeConstants.DECIMAL_TYPE_NAME + "\", but \"" + argumentOIs[0].getTypeName() + "\" was found."); } return resultText; Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java (original) +++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInFile.java Tue Jul 15 18:50:51 2014 @@ -33,8 +33,8 @@ import org.apache.hadoop.hive.ql.exec.UD import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; /** * IN_FILE(str, filename) returns true if 'str' appears in the file specified @@ -59,29 +59,35 @@ public class GenericUDFInFile extends Ge "IN_FILE() accepts exactly 2 arguments."); } - for (int i = 0; i < arguments.length; i++) { - if (!String.class.equals( - PrimitiveObjectInspectorUtils. - getJavaPrimitiveClassFromObjectInspector(arguments[i]))) { - throw new UDFArgumentTypeException(i, "The " - + GenericUDFUtils.getOrdinal(i + 1) - + " argument of function IN_FILE must be a string but " - + arguments[i].toString() + " was given."); - } - } - strObjectInspector = arguments[0]; fileObjectInspector = arguments[1]; - if (!ObjectInspectorUtils.isConstantObjectInspector(fileObjectInspector)) { - throw new UDFArgumentTypeException(1, - "The second argument of IN_FILE() must be a constant string but " + - fileObjectInspector.toString() + " was given."); + if (!isTypeCompatible(strObjectInspector)) { + throw new UDFArgumentTypeException(0, "The first " + + "argument of function IN_FILE must be a string, " + + "char or varchar but " + + strObjectInspector.toString() + " was given."); + } + + if (((PrimitiveObjectInspector) fileObjectInspector).getPrimitiveCategory() != + PrimitiveObjectInspector.PrimitiveCategory.STRING || + !ObjectInspectorUtils.isConstantObjectInspector(fileObjectInspector)) { + throw new UDFArgumentTypeException(1, "The second " + + "argument of IN_FILE() must be a constant string but " + + fileObjectInspector.toString() + " was given."); } return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector; } + private boolean isTypeCompatible(ObjectInspector argument) { + PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) argument); + return + poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.STRING || + poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR || + poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR; + } + @Override public String[] getRequiredFiles() { return new String[] { @@ -96,12 +102,12 @@ public class GenericUDFInFile extends Ge return null; } - String str = (String)ObjectInspectorUtils.copyToStandardJavaObject( - arguments[0].get(), strObjectInspector); + String str = ObjectInspectorUtils.copyToStandardJavaObject( + arguments[0].get(), strObjectInspector).toString(); if (set == null) { String fileName = (String)ObjectInspectorUtils.copyToStandardJavaObject( - arguments[1].get(), fileObjectInspector); + arguments[1].get(), fileObjectInspector); try { load(new FileInputStream((new File(fileName)).getName())); } catch (FileNotFoundException e) { Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant1.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant1.q?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant1.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_role_grant1.q Tue Jul 15 18:50:51 2014 @@ -26,6 +26,10 @@ create role src_role_wadmin; grant src_role_wadmin to user user2 with admin option; show role grant user user2; +-- revoke admin option +revoke admin option for src_role_wadmin from user user2; +show role grant user user2; + -- revoke role without role keyword revoke src_role_wadmin from user user2; show role grant user user2; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_exists_having.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_exists_having.q?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_exists_having.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_exists_having.q Tue Jul 15 18:50:51 2014 @@ -1,4 +1,4 @@ - +set hive.optimize.correlation=false; -- no agg, corr explain @@ -22,6 +22,30 @@ having exists ) ; +set hive.optimize.correlation=true; + +-- no agg, corr +explain +select b.key, count(*) +from src b +group by b.key +having exists + (select a.key + from src a + where a.key = b.key and a.value > 'val_9' + ) +; + +select b.key, count(*) +from src b +group by b.key +having exists + (select a.key + from src a + where a.key = b.key and a.value > 'val_9' + ) +; + -- view test create view cv1 as select b.key, count(*) as c Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_in_having.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_in_having.q?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_in_having.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/subquery_in_having.q Tue Jul 15 18:50:51 2014 @@ -40,6 +40,8 @@ group by key, value having count(*) in (select count(*) from src s1 where s1.key > '9' and s1.value = b.value group by s1.key ) ; +set hive.optimize.correlation=false; + -- agg, non corr explain select p_mfgr, avg(p_size) @@ -53,6 +55,21 @@ having b.p_mfgr in ) ; +set hive.optimize.correlation=true; + +-- agg, non corr +explain +select p_mfgr, avg(p_size) +from part b +group by b.p_mfgr +having b.p_mfgr in + (select p_mfgr + from part + group by p_mfgr + having max(p_size) - min(p_size) < 20 + ) +; + -- join on agg select b.key, min(b.value) from src b Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/truncate_table.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/truncate_table.q?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/truncate_table.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/truncate_table.q Tue Jul 15 18:50:51 2014 @@ -12,24 +12,31 @@ load data local inpath '../../data/files load data local inpath '../../data/files/kv1.txt' into table srcpart_truncate partition (ds='2008-04-09', hr='11'); load data local inpath '../../data/files/kv1.txt' into table srcpart_truncate partition (ds='2008-04-09', hr='12'); +analyze table src_truncate compute statistics; +analyze table srcpart_truncate partition(ds,hr) compute statistics; set hive.fetch.task.conversion=more; +set hive.compute.query.using.stats=true; -- truncate non-partitioned table explain TRUNCATE TABLE src_truncate; TRUNCATE TABLE src_truncate; select * from src_truncate; +select count (*) from src_truncate; -- truncate a partition explain TRUNCATE TABLE srcpart_truncate partition (ds='2008-04-08', hr='11'); TRUNCATE TABLE srcpart_truncate partition (ds='2008-04-08', hr='11'); select * from srcpart_truncate where ds='2008-04-08' and hr='11'; +select count(*) from srcpart_truncate where ds='2008-04-08' and hr='11'; -- truncate partitions with partial spec explain TRUNCATE TABLE srcpart_truncate partition (ds, hr='12'); TRUNCATE TABLE srcpart_truncate partition (ds, hr='12'); select * from srcpart_truncate where hr='12'; +select count(*) from srcpart_truncate where hr='12'; -- truncate partitioned table explain TRUNCATE TABLE srcpart_truncate; TRUNCATE TABLE srcpart_truncate; select * from srcpart_truncate; +select count(*) from srcpart_truncate; Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/udf_format_number.q Tue Jul 15 18:50:51 2014 @@ -63,3 +63,11 @@ SELECT format_number(CAST(12332.123456 A format_number(CAST(12332.1 AS FLOAT), 4), format_number(CAST(-12332.2 AS FLOAT), 0) FROM src tablesample (1 rows); + +-- decimals +SELECT format_number(12332.123456BD, 4), + format_number(12332.123456BD, 2), + format_number(12332.1BD, 4), + format_number(-12332.2BD, 0), + format_number(CAST(12332.567 AS DECIMAL(8, 1)), 4) +FROM src tablesample (1 rows); Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/udf_in_file.q URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/udf_in_file.q?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/queries/clientpositive/udf_in_file.q (original) +++ hive/branches/cbo/ql/src/test/queries/clientpositive/udf_in_file.q Tue Jul 15 18:50:51 2014 @@ -1,12 +1,30 @@ DESCRIBE FUNCTION in_file; +CREATE TABLE value_src (str_val char(3), ch_val STRING, vch_val varchar(10), + str_val_neg char(3), ch_val_neg STRING, vch_val_neg varchar(10)) + ROW FORMAT DELIMITED FIELDS TERMINATED BY ','; + +LOAD DATA LOCAL INPATH '../../data/files/in_file.dat' INTO TABLE value_src; + EXPLAIN -SELECT in_file("303", "../../data/files/test2.dat"), +SELECT in_file(str_val, "../../data/files/test2.dat"), + in_file(ch_val, "../../data/files/test2.dat"), + in_file(vch_val, "../../data/files/test2.dat"), + in_file(str_val_neg, "../../data/files/test2.dat"), + in_file(ch_val_neg, "../../data/files/test2.dat"), + in_file(vch_val_neg, "../../data/files/test2.dat"), + in_file("303", "../../data/files/test2.dat"), in_file("304", "../../data/files/test2.dat"), in_file(CAST(NULL AS STRING), "../../data/files/test2.dat") -FROM src LIMIT 1; +FROM value_src LIMIT 1; -SELECT in_file("303", "../../data/files/test2.dat"), +SELECT in_file(str_val, "../../data/files/test2.dat"), + in_file(ch_val, "../../data/files/test2.dat"), + in_file(vch_val, "../../data/files/test2.dat"), + in_file(str_val_neg, "../../data/files/test2.dat"), + in_file(ch_val_neg, "../../data/files/test2.dat"), + in_file(vch_val_neg, "../../data/files/test2.dat"), + in_file("303", "../../data/files/test2.dat"), in_file("304", "../../data/files/test2.dat"), in_file(CAST(NULL AS STRING), "../../data/files/test2.dat") -FROM src LIMIT 1; +FROM value_src LIMIT 1; \ No newline at end of file Modified: hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out (original) +++ hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out Tue Jul 15 18:50:51 2014 @@ -103,3 +103,7 @@ Stripes: Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DIRECT_V2 + +File length: 1932446 bytes +Padding length: 0 bytes +Padding ratio: 0% Modified: hive/branches/cbo/ql/src/test/resources/orc-file-dump.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/resources/orc-file-dump.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/resources/orc-file-dump.out (original) +++ hive/branches/cbo/ql/src/test/resources/orc-file-dump.out Tue Jul 15 18:50:51 2014 @@ -108,3 +108,7 @@ Stripes: Encoding column 1: DIRECT_V2 Encoding column 2: DIRECT_V2 Encoding column 3: DICTIONARY_V2 + +File length: 269529 bytes +Padding length: 0 bytes +Padding ratio: 0% Modified: hive/branches/cbo/ql/src/test/results/clientnegative/authorization_role_grant.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientnegative/authorization_role_grant.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientnegative/authorization_role_grant.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientnegative/authorization_role_grant.q.out Tue Jul 15 18:50:51 2014 @@ -41,4 +41,4 @@ POSTHOOK: query: set role role_noadmin POSTHOOK: type: SHOW_ROLES PREHOOK: query: grant src_role_wadmin to user user3 PREHOOK: type: GRANT_ROLE -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException: Current user : user2 is not allowed to grant role. User has to belong to ADMIN role and have it as current role, for this action. Otherwise, grantor need to have ADMIN privileges on role being granted and have it as a current role for this action. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAccessControlException: Current user : user2 is not allowed to grant role. User has to belong to ADMIN role and have it as current role, for this action. Otherwise, grantor need to have ADMIN OPTION on role being granted and have it as a current role for this action. Modified: hive/branches/cbo/ql/src/test/results/clientnegative/fileformat_bad_class.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientnegative/fileformat_bad_class.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientnegative/fileformat_bad_class.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientnegative/fileformat_bad_class.q.out Tue Jul 15 18:50:51 2014 @@ -1 +1 @@ -FAILED: SemanticException [Error 10055]: Output Format must implement HiveOutputFormat, otherwise it should be either IgnoreKeyTextOutputFormat or SequenceFileOutputFormat +FAILED: SemanticException Cannot find class 'ClassDoesNotExist' Modified: hive/branches/cbo/ql/src/test/results/clientnegative/genericFileFormat.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientnegative/genericFileFormat.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientnegative/genericFileFormat.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientnegative/genericFileFormat.q.out Tue Jul 15 18:50:51 2014 @@ -1 +1 @@ -FAILED: SemanticException Unrecognized file format in STORED AS clause: foo +FAILED: SemanticException Unrecognized file format in STORED AS clause: 'FOO' Modified: hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong5.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong5.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong5.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong5.q.out Tue Jul 15 18:50:51 2014 @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 2:21 Argument type mismatch '321.23': Argument 1 of function FORMAT_NUMBER must be "tinyint" or "smallint" or "int" or "bigint" or "double" or "float", but "array" was found. +FAILED: SemanticException [Error 10016]: Line 2:21 Argument type mismatch '321.23': Argument 1 of function FORMAT_NUMBER must be "tinyint" or "smallint" or "int" or "bigint" or "double" or "float" or "decimal", but "array" was found. Modified: hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong7.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong7.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong7.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientnegative/udf_format_number_wrong7.q.out Tue Jul 15 18:50:51 2014 @@ -1 +1 @@ -FAILED: SemanticException [Error 10016]: Line 2:21 Argument type mismatch '4': Argument 1 of function FORMAT_NUMBER must be "tinyint" or "smallint" or "int" or "bigint" or "double" or "float", but "string" was found. +FAILED: SemanticException [Error 10016]: Line 2:21 Argument type mismatch '4': Argument 1 of function FORMAT_NUMBER must be "tinyint" or "smallint" or "int" or "bigint" or "double" or "float" or "decimal", but "string" was found. Modified: hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant1.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant1.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant1.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/authorization_role_grant1.q.out Tue Jul 15 18:50:51 2014 @@ -71,6 +71,18 @@ POSTHOOK: query: show role grant user us POSTHOOK: type: SHOW_ROLE_GRANT public false -1 src_role_wadmin true -1 hive_admin_user +PREHOOK: query: -- revoke admin option +revoke admin option for src_role_wadmin from user user2 +PREHOOK: type: REVOKE_ROLE +POSTHOOK: query: -- revoke admin option +revoke admin option for src_role_wadmin from user user2 +POSTHOOK: type: REVOKE_ROLE +PREHOOK: query: show role grant user user2 +PREHOOK: type: SHOW_ROLE_GRANT +POSTHOOK: query: show role grant user user2 +POSTHOOK: type: SHOW_ROLE_GRANT +public false -1 +src_role_wadmin false -1 hive_admin_user PREHOOK: query: -- revoke role without role keyword revoke src_role_wadmin from user user2 PREHOOK: type: REVOKE_ROLE Modified: hive/branches/cbo/ql/src/test/results/clientpositive/create_union_table.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/create_union_table.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/create_union_table.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/create_union_table.q.out Tue Jul 15 18:50:51 2014 @@ -14,6 +14,7 @@ STAGE PLANS: columns: mydata uniontype,struct>, strct struct input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: abc PREHOOK: query: create table abc(mydata uniontype,struct>, Modified: hive/branches/cbo/ql/src/test/results/clientpositive/ctas.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/ctas.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/ctas.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/ctas.q.out Tue Jul 15 18:50:51 2014 @@ -95,6 +95,7 @@ STAGE PLANS: columns: k string, value string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: nzhang_CTAS1 Stage: Stage-3 @@ -239,6 +240,7 @@ STAGE PLANS: columns: key string, value string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: nzhang_ctas2 Stage: Stage-3 @@ -594,6 +596,7 @@ STAGE PLANS: field delimiter: , input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: nzhang_ctas4 Stage: Stage-3 @@ -677,7 +680,8 @@ TOK_CREATETABLE ',' TOK_TABLEROWFORMATLINES '\012' - TOK_TBLTEXTFILE + TOK_FILEFORMAT_GENERIC + textfile TOK_QUERY TOK_FROM TOK_TABREF @@ -891,6 +895,7 @@ STAGE PLANS: line delimiter: output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: nzhang_ctas5 Stage: Stage-3 Modified: hive/branches/cbo/ql/src/test/results/clientpositive/ctas_colname.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/ctas_colname.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/ctas_colname.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/ctas_colname.q.out Tue Jul 15 18:50:51 2014 @@ -71,6 +71,7 @@ STAGE PLANS: columns: _col0 double, _col1 bigint, _c1 double, _c2 bigint input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: summary Stage: Stage-2 @@ -187,6 +188,7 @@ STAGE PLANS: columns: key string, value string, rr int input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: x4 Stage: Stage-2 @@ -351,6 +353,7 @@ STAGE PLANS: columns: key string, value string, lead1 string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: x5 Stage: Stage-3 @@ -493,6 +496,7 @@ STAGE PLANS: columns: _col0 string, _c1 string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: x6 Stage: Stage-2 @@ -614,6 +618,7 @@ STAGE PLANS: columns: _col0 string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: x7 Stage: Stage-2 @@ -1045,6 +1050,7 @@ STAGE PLANS: columns: _col0 string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: x8 Stage: Stage-2 @@ -1178,6 +1184,7 @@ STAGE PLANS: columns: _c0 string, key string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: x9 Stage: Stage-2 Modified: hive/branches/cbo/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out Tue Jul 15 18:50:51 2014 @@ -74,6 +74,7 @@ STAGE PLANS: columns: key string, value string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: table_db1 Stage: Stage-2 Modified: hive/branches/cbo/ql/src/test/results/clientpositive/groupby_duplicate_key.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/groupby_duplicate_key.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/groupby_duplicate_key.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/groupby_duplicate_key.q.out Tue Jul 15 18:50:51 2014 @@ -139,6 +139,7 @@ STAGE PLANS: columns: key string, dummy1 string, dummy2 string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: dummy Stage: Stage-2 Modified: hive/branches/cbo/ql/src/test/results/clientpositive/input15.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/input15.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/input15.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/input15.q.out Tue Jul 15 18:50:51 2014 @@ -15,6 +15,7 @@ STAGE PLANS: field delimiter: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: TEST15 PREHOOK: query: CREATE TABLE TEST15(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE Modified: hive/branches/cbo/ql/src/test/results/clientpositive/inputddl1.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/inputddl1.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/inputddl1.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/inputddl1.q.out Tue Jul 15 18:50:51 2014 @@ -14,6 +14,7 @@ STAGE PLANS: columns: key int, value string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: INPUTDDL1 PREHOOK: query: CREATE TABLE INPUTDDL1(key INT, value STRING) STORED AS TEXTFILE Modified: hive/branches/cbo/ql/src/test/results/clientpositive/inputddl2.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/inputddl2.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/inputddl2.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/inputddl2.q.out Tue Jul 15 18:50:51 2014 @@ -15,6 +15,7 @@ STAGE PLANS: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat partition columns: ds string, country string + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: INPUTDDL2 PREHOOK: query: CREATE TABLE INPUTDDL2(key INT, value STRING) PARTITIONED BY(ds STRING, country STRING) STORED AS TEXTFILE Modified: hive/branches/cbo/ql/src/test/results/clientpositive/inputddl3.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/inputddl3.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/inputddl3.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/inputddl3.q.out Tue Jul 15 18:50:51 2014 @@ -15,6 +15,7 @@ STAGE PLANS: field delimiter: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: INPUTDDL3 PREHOOK: query: CREATE TABLE INPUTDDL3(key INT, value STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE Modified: hive/branches/cbo/ql/src/test/results/clientpositive/merge3.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/merge3.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== Files hive/branches/cbo/ql/src/test/results/clientpositive/merge3.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/merge3.q.out Tue Jul 15 18:50:51 2014 differ Modified: hive/branches/cbo/ql/src/test/results/clientpositive/nonmr_fetch.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/nonmr_fetch.q.out?rev=1610806&r1=1610805&r2=1610806&view=diff ============================================================================== --- hive/branches/cbo/ql/src/test/results/clientpositive/nonmr_fetch.q.out (original) +++ hive/branches/cbo/ql/src/test/results/clientpositive/nonmr_fetch.q.out Tue Jul 15 18:50:51 2014 @@ -1014,6 +1014,7 @@ STAGE PLANS: columns: key string, value string input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: srcx Stage: Stage-2