hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From br...@apache.org
Subject svn commit: r1629544 [4/33] - in /hive/branches/spark-new: ./ accumulo-handler/ beeline/ beeline/src/java/org/apache/hive/beeline/ bin/ bin/ext/ common/ common/src/java/org/apache/hadoop/hive/conf/ common/src/test/org/apache/hadoop/hive/common/type/ co...
Date Sun, 05 Oct 2014 22:26:58 GMT
Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java Sun Oct  5 22:26:43 2014
@@ -171,41 +171,36 @@ public class TestAuthorizationPreEventLi
 
     driver.run("create database " + dbName);
     listSize++;
-    Database db = msc.getDatabase(dbName);
-
     Database dbFromEvent = (Database)assertAndExtractSingleObjectFromEvent(listSize, authCalls,
         DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.DB);
+    Database db = msc.getDatabase(dbName);
     validateCreateDb(db,dbFromEvent);
 
     driver.run("use " + dbName);
     driver.run(String.format("create table %s (a string) partitioned by (b string)", tblName));
-    listSize++;
-    Table tbl = msc.getTable(dbName, tblName);
+    listSize = authCalls.size();
 
     Table tblFromEvent = (
         (org.apache.hadoop.hive.ql.metadata.Table)
         assertAndExtractSingleObjectFromEvent(listSize, authCalls,
             DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.TABLE))
             .getTTable();
+    Table tbl = msc.getTable(dbName, tblName);
     validateCreateTable(tbl, tblFromEvent);
 
     driver.run("alter table tmptbl add partition (b='2011')");
-    listSize++;
-    Partition part = msc.getPartition("hive3705", "tmptbl", "b=2011");
+    listSize = authCalls.size();
 
     Partition ptnFromEvent = (
         (org.apache.hadoop.hive.ql.metadata.Partition)
         assertAndExtractSingleObjectFromEvent(listSize, authCalls,
             DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.PARTITION))
             .getTPartition();
+    Partition part = msc.getPartition("hive3705", "tmptbl", "b=2011");
     validateAddPartition(part,ptnFromEvent);
 
     driver.run(String.format("alter table %s touch partition (%s)", tblName, "b='2011'"));
-    listSize++;
-
-    //the partition did not change,
-    // so the new partition should be similar to the original partition
-    Partition modifiedP = msc.getPartition(dbName, tblName, "b=2011");
+    listSize = authCalls.size();
 
     Partition ptnFromEventAfterAlter = (
         (org.apache.hadoop.hive.ql.metadata.Partition)
@@ -213,6 +208,9 @@ public class TestAuthorizationPreEventLi
             DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.PARTITION))
             .getTPartition();
 
+    //the partition did not change,
+    // so the new partition should be similar to the original partition
+    Partition modifiedP = msc.getPartition(dbName, tblName, "b=2011");
     validateAlterPartition(part, modifiedP, ptnFromEventAfterAlter.getDbName(),
         ptnFromEventAfterAlter.getTableName(), ptnFromEventAfterAlter.getValues(),
         ptnFromEventAfterAlter);
@@ -220,8 +218,9 @@ public class TestAuthorizationPreEventLi
 
     List<String> part_vals = new ArrayList<String>();
     part_vals.add("c=2012");
-    Partition newPart = msc.appendPartition(dbName, tblName, part_vals);
 
+    listSize = authCalls.size();
+    Partition newPart = msc.appendPartition(dbName, tblName, part_vals);
     listSize++;
 
     Partition newPtnFromEvent = (
@@ -233,25 +232,23 @@ public class TestAuthorizationPreEventLi
 
 
     driver.run(String.format("alter table %s rename to %s", tblName, renamed));
-    listSize++;
+    listSize = authCalls.size();
 
-    Table renamedTable = msc.getTable(dbName, renamed);
     Table renamedTableFromEvent = (
         (org.apache.hadoop.hive.ql.metadata.Table)
         assertAndExtractSingleObjectFromEvent(listSize, authCalls,
             DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.TABLE))
             .getTTable();
 
+    Table renamedTable = msc.getTable(dbName, renamed);
     validateAlterTable(tbl, renamedTable, renamedTableFromEvent,
         renamedTable);
     assertFalse(tbl.getTableName().equals(renamedTable.getTableName()));
 
     //change the table name back
     driver.run(String.format("alter table %s rename to %s", renamed, tblName));
-    listSize++;
-
     driver.run(String.format("alter table %s drop partition (b='2011')", tblName));
-    listSize++;
+    listSize = authCalls.size();
 
     Partition ptnFromDropPartition = (
         (org.apache.hadoop.hive.ql.metadata.Partition)
@@ -262,7 +259,7 @@ public class TestAuthorizationPreEventLi
     validateDropPartition(modifiedP, ptnFromDropPartition);
 
     driver.run("drop table " + tblName);
-    listSize++;
+    listSize = authCalls.size();
     Table tableFromDropTableEvent = (
         (org.apache.hadoop.hive.ql.metadata.Table)
         assertAndExtractSingleObjectFromEvent(listSize, authCalls,
@@ -290,16 +287,16 @@ public class TestAuthorizationPreEventLi
     }
 
     tCustom.setTableName(tbl.getTableName() + "_custom");
+    listSize = authCalls.size();
     msc.createTable(tCustom);
     listSize++;
 
-    Table customCreatedTable = msc.getTable(tCustom.getDbName(), tCustom.getTableName());
     Table customCreatedTableFromEvent = (
         (org.apache.hadoop.hive.ql.metadata.Table)
             assertAndExtractSingleObjectFromEvent(listSize, authCalls,
                 DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.TABLE))
         .getTTable();
-
+    Table customCreatedTable = msc.getTable(tCustom.getDbName(), tCustom.getTableName());
     validateCreateTable(tCustom,customCreatedTable);
     validateCreateTable(tCustom,customCreatedTableFromEvent);
 
@@ -316,8 +313,10 @@ public class TestAuthorizationPreEventLi
     assertEquals(tCustom.getSd().getSerdeInfo().getSerializationLib(),
         customCreatedTableFromEvent.getSd().getSerdeInfo().getSerializationLib());
 
-    msc.dropTable(tCustom.getDbName(),tCustom.getTableName());
-    listSize++;
+    listSize = authCalls.size();
+    msc.dropTable(tCustom.getDbName(), tCustom.getTableName());
+    listSize += 2;
+
     Table table2FromDropTableEvent = (
         (org.apache.hadoop.hive.ql.metadata.Table)
             assertAndExtractSingleObjectFromEvent(listSize, authCalls,
@@ -327,7 +326,7 @@ public class TestAuthorizationPreEventLi
     validateDropTable(tCustom, table2FromDropTableEvent);
 
     driver.run("drop database " + dbName);
-    listSize++;
+    listSize = authCalls.size();
     Database dbFromDropDatabaseEvent =
         (Database)assertAndExtractSingleObjectFromEvent(listSize, authCalls,
         DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.DB);

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java Sun Oct  5 22:26:43 2014
@@ -89,6 +89,7 @@ public class TestMetastoreAuthorizationP
         AuthorizationPreEventListener.class.getName());
     System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname,
         getAuthorizationProvider());
+    setupMetaStoreReadAuthorization();
     System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
         InjectableDummyAuthenticator.class.getName());
     System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, "");
@@ -115,6 +116,13 @@ public class TestMetastoreAuthorizationP
     driver = new Driver(clientHiveConf);
   }
 
+  protected void setupMetaStoreReadAuthorization() {
+    // read authorization does not work with default/legacy authorization mode
+    // It is a chicken and egg problem granting select privilege to database, as the
+    // grant statement would invoke get_database which needs select privilege
+    System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS.varname, "false");
+  }
+
   @Override
   protected void tearDown() throws Exception {
     super.tearDown();

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java Sun Oct  5 22:26:43 2014
@@ -88,6 +88,7 @@ public class TestMultiAuthorizationPreEv
 
     // verify that the actual action also went through
     Database db = msc.getDatabase(dbName);
+    listSize += 2; // 1 read database auth calls for each authorization provider
     Database dbFromEvent = (Database)assertAndExtractSingleObjectFromEvent(listSize, authCalls,
         DummyHiveMetastoreAuthorizationProvider.AuthCallContextType.DB);
     validateCreateDb(db,dbFromEvent);

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationDrops.java Sun Oct  5 22:26:43 2014
@@ -18,88 +18,19 @@
 
 package org.apache.hadoop.hive.ql.security;
 
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hive.cli.CliSessionState;
-import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener;
-import org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
+import org.junit.Test;
 
 /**
  * Test cases focusing on drop table permission checks
  */
-public class TestStorageBasedMetastoreAuthorizationDrops extends TestCase{
-  protected HiveConf clientHiveConf;
-  protected HiveMetaStoreClient msc;
-  protected Driver driver;
-  protected UserGroupInformation ugi;
-  private static int objNum = 0;
-
-  protected String getAuthorizationProvider(){
-    return StorageBasedAuthorizationProvider.class.getName();
-  }
-
-  protected HiveConf createHiveConf() throws Exception {
-    return new HiveConf(this.getClass());
-  }
-
-  @Override
-  protected void setUp() throws Exception {
-
-    super.setUp();
-
-    int port = MetaStoreUtils.findFreePort();
-
-    // Turn on metastore-side authorization
-    System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname,
-        AuthorizationPreEventListener.class.getName());
-    System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_MANAGER.varname,
-        getAuthorizationProvider());
-    System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname,
-        InjectableDummyAuthenticator.class.getName());
-
-    MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
-
-    clientHiveConf = createHiveConf();
-
-    // Turn off client-side authorization
-    clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false);
-
-    clientHiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
-    clientHiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
-    clientHiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
-
-    clientHiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
-    clientHiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
-
-    ugi = ShimLoader.getHadoopShims().getUGIForConf(clientHiveConf);
-
-    SessionState.start(new CliSessionState(clientHiveConf));
-    msc = new HiveMetaStoreClient(clientHiveConf, null);
-    driver = new Driver(clientHiveConf);
-
-    setupFakeUser();
-    InjectableDummyAuthenticator.injectMode(false);
-  }
-
+public class TestStorageBasedMetastoreAuthorizationDrops extends StorageBasedMetastoreTestBase {
 
+  @Test
   public void testDropDatabase() throws Exception {
     dropDatabaseByOtherUser("-rwxrwxrwx", 0);
     dropDatabaseByOtherUser("-rwxrwxrwt", 1);
@@ -111,12 +42,12 @@ public class TestStorageBasedMetastoreAu
    * @param expectedRet - expected return code for drop by other user
    * @throws Exception
    */
-  private void dropDatabaseByOtherUser(String perm, int expectedRet) throws Exception {
+  public void dropDatabaseByOtherUser(String perm, int expectedRet) throws Exception {
     String dbName = getTestDbName();
     setPermissions(clientHiveConf.getVar(ConfVars.METASTOREWAREHOUSE), perm);
 
     CommandProcessorResponse resp = driver.run("create database " + dbName);
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
     Database db = msc.getDatabase(dbName);
     validateCreateDb(db, dbName);
 
@@ -124,10 +55,11 @@ public class TestStorageBasedMetastoreAu
 
 
     resp = driver.run("drop database " + dbName);
-    assertEquals(expectedRet, resp.getResponseCode());
+    Assert.assertEquals(expectedRet, resp.getResponseCode());
 
   }
 
+  @Test
   public void testDropTable() throws Exception {
     dropTableByOtherUser("-rwxrwxrwx", 0);
     dropTableByOtherUser("-rwxrwxrwt", 1);
@@ -138,13 +70,13 @@ public class TestStorageBasedMetastoreAu
    * @param expectedRet expected return code on drop table
    * @throws Exception
    */
-  private void dropTableByOtherUser(String perm, int expectedRet) throws Exception {
+  public void dropTableByOtherUser(String perm, int expectedRet) throws Exception {
     String dbName = getTestDbName();
     String tblName = getTestTableName();
     setPermissions(clientHiveConf.getVar(ConfVars.METASTOREWAREHOUSE), "-rwxrwxrwx");
 
     CommandProcessorResponse resp = driver.run("create database " + dbName);
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
     Database db = msc.getDatabase(dbName);
     validateCreateDb(db, dbName);
 
@@ -152,18 +84,19 @@ public class TestStorageBasedMetastoreAu
 
     String dbDotTable = dbName + "." + tblName;
     resp = driver.run("create table " + dbDotTable + "(i int)");
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
 
 
     InjectableDummyAuthenticator.injectMode(true);
     resp = driver.run("drop table " + dbDotTable);
-    assertEquals(expectedRet, resp.getResponseCode());
+    Assert.assertEquals(expectedRet, resp.getResponseCode());
   }
 
   /**
    * Drop view should not be blocked by SBA. View will not have any location to drop.
    * @throws Exception
    */
+  @Test
   public void testDropView() throws Exception {
     String dbName = getTestDbName();
     String tblName = getTestTableName();
@@ -171,7 +104,7 @@ public class TestStorageBasedMetastoreAu
     setPermissions(clientHiveConf.getVar(ConfVars.METASTOREWAREHOUSE), "-rwxrwxrwx");
 
     CommandProcessorResponse resp = driver.run("create database " + dbName);
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
     Database db = msc.getDatabase(dbName);
     validateCreateDb(db, dbName);
 
@@ -179,20 +112,20 @@ public class TestStorageBasedMetastoreAu
 
     String dbDotTable = dbName + "." + tblName;
     resp = driver.run("create table " + dbDotTable + "(i int)");
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
 
     String dbDotView = dbName + "." + viewName;
     resp = driver.run("create view " + dbDotView + " as select * from " +  dbDotTable);
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
 
     resp = driver.run("drop view " + dbDotView);
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
 
     resp = driver.run("drop table " + dbDotTable);
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
   }
 
-
+  @Test
   public void testDropPartition() throws Exception {
     dropPartitionByOtherUser("-rwxrwxrwx", 0);
     dropPartitionByOtherUser("-rwxrwxrwt", 1);
@@ -203,70 +136,29 @@ public class TestStorageBasedMetastoreAu
    * @param expectedRet expected return code
    * @throws Exception
    */
-  private void dropPartitionByOtherUser(String perm, int expectedRet) throws Exception {
+  public void dropPartitionByOtherUser(String perm, int expectedRet) throws Exception {
     String dbName = getTestDbName();
     String tblName = getTestTableName();
     setPermissions(clientHiveConf.getVar(ConfVars.METASTOREWAREHOUSE), "-rwxrwxrwx");
 
     CommandProcessorResponse resp = driver.run("create database " + dbName);
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
     Database db = msc.getDatabase(dbName);
     validateCreateDb(db, dbName);
     setPermissions(db.getLocationUri(), "-rwxrwxrwx");
 
     String dbDotTable = dbName + "." + tblName;
     resp = driver.run("create table " + dbDotTable + "(i int) partitioned by (b string)");
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
     Table tab = msc.getTable(dbName, tblName);
     setPermissions(tab.getSd().getLocation(), perm);
 
     resp = driver.run("alter table " + dbDotTable + " add partition (b='2011')");
-    assertEquals(0, resp.getResponseCode());
+    Assert.assertEquals(0, resp.getResponseCode());
 
     InjectableDummyAuthenticator.injectMode(true);
     resp = driver.run("alter table " + dbDotTable + " drop partition (b='2011')");
-    assertEquals(expectedRet, resp.getResponseCode());
+    Assert.assertEquals(expectedRet, resp.getResponseCode());
   }
 
-  private void setupFakeUser() {
-    String fakeUser = "mal";
-    List<String> fakeGroupNames = new ArrayList<String>();
-    fakeGroupNames.add("groupygroup");
-
-    InjectableDummyAuthenticator.injectUserName(fakeUser);
-    InjectableDummyAuthenticator.injectGroupNames(fakeGroupNames);
-  }
-
-  private String setupUser() {
-    return ugi.getUserName();
-  }
-
-  private String getTestTableName() {
-    return this.getClass().getSimpleName() + "tab" + ++objNum;
-  }
-
-  private String getTestDbName() {
-    return this.getClass().getSimpleName() + "db" + ++objNum;
-  }
-
-  @Override
-  protected void tearDown() throws Exception {
-    super.tearDown();
-    InjectableDummyAuthenticator.injectMode(false);
-  }
-
-  protected void setPermissions(String locn, String permissions) throws Exception {
-    FileSystem fs = FileSystem.get(new URI(locn), clientHiveConf);
-    fs.setPermission(new Path(locn), FsPermission.valueOf(permissions));
-  }
-
-  private void validateCreateDb(Database expectedDb, String dbName) {
-    assertEquals(expectedDb.getName().toLowerCase(), dbName.toLowerCase());
-  }
-
-  private void validateCreateTable(Table expectedTable, String tblName, String dbName) {
-    assertNotNull(expectedTable);
-    assertEquals(expectedTable.getTableName().toLowerCase(),tblName.toLowerCase());
-    assertEquals(expectedTable.getDbName().toLowerCase(),dbName.toLowerCase());
-  }
 }

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProvider.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProvider.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestStorageBasedMetastoreAuthorizationProvider.java Sun Oct  5 22:26:43 2014
@@ -23,6 +23,7 @@ import java.net.URI;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider;
 
@@ -102,4 +103,10 @@ public class TestStorageBasedMetastoreAu
     return super.getTestTableName() + "_SBAP";
   }
 
+  @Override
+  protected void setupMetaStoreReadAuthorization() {
+    // enable read authorization in metastore
+    System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHORIZATION_AUTH_READS.varname, "true");
+  }
+
 }

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java Sun Oct  5 22:26:43 2014
@@ -98,7 +98,7 @@ public class TestHiveAuthorizerCheckInvo
     runCmd("create database " + dbName);
     // Need a separate table for ACID testing since it has to be bucketed and it has to be Acid
     runCmd("create table " + acidTableName + " (i int, j int) clustered by (i) into 2 buckets " +
-        "stored as orc");
+        "stored as orc TBLPROPERTIES ('transactional'='true')");
   }
 
   private static void runCmd(String cmd) throws CommandNeedRetryException {

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestHadoop20SAuthBridge.java Sun Oct  5 22:26:43 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.security.SaslRp
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
 import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
 import org.apache.hadoop.security.authorize.ProxyUsers;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.Token;
@@ -129,7 +130,7 @@ public class TestHadoop20SAuthBridge ext
     }
     builder.append("127.0.1.1,");
     builder.append(InetAddress.getLocalHost().getCanonicalHostName());
-    conf.setStrings(ProxyUsers.getProxySuperuserIpConfKey(superUserShortName),
+    conf.setStrings(DefaultImpersonationProvider.getProxySuperuserIpConfKey(superUserShortName),
         builder.toString());
   }
 
@@ -292,7 +293,7 @@ public class TestHadoop20SAuthBridge ext
   private void setGroupsInConf(String[] groupNames, String proxyUserName)
   throws IOException {
    conf.set(
-      ProxyUsers.getProxySuperuserGroupConfKey(proxyUserName),
+      DefaultImpersonationProvider.getProxySuperuserGroupConfKey(proxyUserName),
       StringUtils.join(",", Arrays.asList(groupNames)));
     configureSuperUserIPAddresses(conf, proxyUserName);
     ProxyUsers.refreshSuperUserGroupsConfiguration(conf);

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java Sun Oct  5 22:26:43 2014
@@ -477,4 +477,31 @@ public class TestBeeLineWithArgs {
     final String EXPECTED_PATTERN = "embedded_table";
     testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
   }
+
+  /**
+   * Test Beeline could show the query progress for time-consuming query.
+   * @throws Throwable
+   */
+  @Test
+  public void testQueryProgress() throws Throwable {
+    final String TEST_NAME = "testQueryProgress";
+    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+        "select count(*) from " + tableName + ";\n";
+    final String EXPECTED_PATTERN = "Parsing command";
+    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL));
+  }
+
+  /**
+   * Test Beeline will hide the query progress when silent option is set.
+   * @throws Throwable
+   */
+  @Test
+  public void testQueryProgressHidden() throws Throwable {
+    final String TEST_NAME = "testQueryProgress";
+    final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+        "!set silent true\n" +
+        "select count(*) from " + tableName + ";\n";
+    final String EXPECTED_PATTERN = "Parsing command";
+    testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(JDBC_URL));
+  }
 }

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Sun Oct  5 22:26:43 2014
@@ -1318,6 +1318,42 @@ public class TestJdbcDriver2 {
   }
 
   @Test
+  public void testResultSetColumnNameCaseInsensitive() throws SQLException {
+    Statement stmt = con.createStatement();
+    ResultSet res;
+
+    res = stmt.executeQuery("select c1 from " + dataTypeTableName + " limit 1");
+    try {
+      int count = 0;
+      while (res.next()) {
+        res.findColumn("c1");
+        res.findColumn("C1");
+        count++;
+      }
+      assertEquals(count, 1);
+    } catch (Exception e) {
+      String msg = "Unexpected exception: " + e;
+      LOG.info(msg, e);
+      fail(msg);
+    }
+
+    res = stmt.executeQuery("select c1 C1 from " + dataTypeTableName + " limit 1");
+    try {
+      int count = 0;
+      while (res.next()) {
+        res.findColumn("c1");
+        res.findColumn("C1");
+        count++;
+      }
+      assertEquals(count, 1);
+    } catch (Exception e) {
+      String msg = "Unexpected exception: " + e;
+      LOG.info(msg, e);
+      fail(msg);
+    }
+  }
+
+  @Test
   public void testResultSetMetaData() throws SQLException {
     Statement stmt = con.createStatement();
 
@@ -2130,4 +2166,82 @@ public class TestJdbcDriver2 {
     }
     stmt.close();
   }
+
+  /**
+   * Test getting query log method in Jdbc
+   * @throws Exception
+   */
+  @Test
+  public void testGetQueryLog() throws Exception {
+    // Prepare
+    String[] expectedLogs = {
+        "Parsing command",
+        "Parse Completed",
+        "Starting Semantic Analysis",
+        "Semantic Analysis Completed",
+        "Starting command"
+    };
+    String sql = "select count(*) from " + tableName;
+
+    // Verify the fetched log (from the beginning of log file)
+    HiveStatement stmt = (HiveStatement)con.createStatement();
+    assertNotNull("Statement is null", stmt);
+    stmt.executeQuery(sql);
+    List<String> logs = stmt.getQueryLog(false, 10000);
+    stmt.close();
+    verifyFetchedLog(logs, expectedLogs);
+
+    // Verify the fetched log (incrementally)
+    final HiveStatement statement = (HiveStatement)con.createStatement();
+    assertNotNull("Statement is null", statement);
+    statement.setFetchSize(10000);
+    final List<String> incrementalLogs = new ArrayList<String>();
+
+    Runnable logThread = new Runnable() {
+      @Override
+      public void run() {
+        while (statement.hasMoreLogs()) {
+          try {
+            incrementalLogs.addAll(statement.getQueryLog());
+            Thread.sleep(500);
+          } catch (SQLException e) {
+            LOG.error("Failed getQueryLog. Error message: " + e.getMessage());
+            fail("error in getting log thread");
+          } catch (InterruptedException e) {
+            LOG.error("Getting log thread is interrupted. Error message: " + e.getMessage());
+            fail("error in getting log thread");
+          }
+        }
+      }
+    };
+
+    Thread thread = new Thread(logThread);
+    thread.setDaemon(true);
+    thread.start();
+    statement.executeQuery(sql);
+    thread.interrupt();
+    thread.join(10000);
+    // fetch remaining logs
+    List<String> remainingLogs;
+    do {
+      remainingLogs = statement.getQueryLog();
+      incrementalLogs.addAll(remainingLogs);
+    } while (remainingLogs.size() > 0);
+    statement.close();
+
+    verifyFetchedLog(incrementalLogs, expectedLogs);
+  }
+
+  private void verifyFetchedLog(List<String> logs, String[] expectedLogs) {
+    StringBuilder stringBuilder = new StringBuilder();
+
+    for (String log : logs) {
+      stringBuilder.append(log);
+    }
+
+    String accumulatedLogs = stringBuilder.toString();
+    for (String expectedLog : expectedLogs) {
+      assertTrue(accumulatedLogs.contains(expectedLog));
+    }
+  }
 }

Modified: hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java (original)
+++ hive/branches/spark-new/itests/hive-unit/src/test/java/org/apache/hive/jdbc/authorization/TestHS2AuthzContext.java Sun Oct  5 22:26:43 2014
@@ -120,9 +120,10 @@ public class TestHS2AuthzContext {
     HiveAuthzContext context = contextCapturer.getValue();
 
     assertEquals("Command ", ctxCmd, context.getCommandString());
-    assertTrue("ip address pattern check", context.getIpAddress().contains("."));
+    assertTrue("ip address pattern check", context.getIpAddress().matches("[.:a-fA-F0-9]+"));
     // ip address size check - check for something better than non zero
     assertTrue("ip address size check", context.getIpAddress().length() > 7);
+
   }
 
   private Connection getConnection(String userName) throws SQLException {

Modified: hive/branches/spark-new/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/src/test/resources/testconfiguration.properties?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark-new/itests/src/test/resources/testconfiguration.properties Sun Oct  5 22:26:43 2014
@@ -55,6 +55,7 @@ minitez.query.files.shared=alter_merge_2
   bucket2.q,\
   bucket3.q,\
   bucket4.q,\
+  cbo_correctness.q,\
   correlationoptimizer1.q,\
   count.q,\
   create_merge_compressed.q,\
@@ -74,6 +75,7 @@ minitez.query.files.shared=alter_merge_2
   disable_merge_for_bucketing.q,\
   dynpart_sort_opt_vectorization.q,\
   dynpart_sort_optimization.q,\
+  dynpart_sort_optimization2.q,\
   enforce_order.q,\
   filter_join_breaktask.q,\
   filter_join_breaktask2.q,\
@@ -116,6 +118,7 @@ minitez.query.files.shared=alter_merge_2
   orc_merge7.q,\
   orc_merge_incompat1.q,\
   orc_merge_incompat2.q,\
+  orc_vectorization_ppd.q,\
   parallel.q,\
   ptf.q,\
   sample1.q,\
@@ -152,24 +155,73 @@ minitez.query.files.shared=alter_merge_2
   update_where_non_partitioned.q,\
   update_where_partitioned.q,\
   update_two_cols.q,\
+  vector_between_in.q,\
   vector_cast_constant.q,\
+  vector_char_4.q,\
+  vector_char_simple.q,\
+  vector_count_distinct.q,\
   vector_data_types.q,\
   vector_decimal_aggregate.q,\
+  vector_distinct_2.q,\
+  vector_elt.q,\
+  vector_groupby_3.q,\
   vector_left_outer_join.q,\
   vector_mapjoin_reduce.q,\
+  vector_non_string_partition.q,\
+  vector_orderby_5.q,\
   vector_string_concat.q,\
+  vector_varchar_4.q,\
+  vector_varchar_simple.q,\
+  vectorization_0.q,\
+  vectorization_1.q,\
+  vectorization_10.q,\
+  vectorization_11.q,\
   vectorization_12.q,\
   vectorization_13.q,\
   vectorization_14.q,\
   vectorization_15.q,\
+  vectorization_16.q,\
+  vectorization_2.q,\
+  vectorization_3.q,\
+  vectorization_4.q,\
+  vectorization_5.q,\
+  vectorization_6.q,\
   vectorization_9.q,\
+  vectorization_decimal_date.q,\
+  vectorization_div0.q,\
+  vectorization_nested_udf.q,\
+  vectorization_not.q,\
+  vectorization_part.q,\
   vectorization_part_project.q,\
+  vectorization_pushdown.q,\
   vectorization_short_regress.q,\
+  vectorized_bucketmapjoin1.q,\
+  vectorized_case.q,\
+  vectorized_context.q,\
   vectorized_mapjoin.q,\
+  vectorized_math_funcs.q,\
   vectorized_nested_mapjoin.q,\
+  vectorized_parquet.q,\
   vectorized_ptf.q,\
+  vectorized_rcfile_columnar.q,\
   vectorized_shufflejoin.q,\
-  vectorized_timestamp_funcs.q
+  vectorized_string_funcs.q,\
+  vectorized_timestamp_funcs.q,\
+  auto_sortmerge_join_1.q,\
+  auto_sortmerge_join_10.q,\
+  auto_sortmerge_join_11.q,\
+  auto_sortmerge_join_12.q,\
+  auto_sortmerge_join_13.q,\
+  auto_sortmerge_join_14.q,\
+  auto_sortmerge_join_15.q,\
+  auto_sortmerge_join_16.q,\
+  auto_sortmerge_join_2.q,\
+  auto_sortmerge_join_3.q,\
+  auto_sortmerge_join_4.q,\
+  auto_sortmerge_join_5.q,\
+  auto_sortmerge_join_7.q,\
+  auto_sortmerge_join_8.q,\
+  auto_sortmerge_join_9.q
 
 minitez.query.files=bucket_map_join_tez1.q,\
   bucket_map_join_tez2.q,\
@@ -186,7 +238,11 @@ minitez.query.files=bucket_map_join_tez1
   tez_joins_explain.q,\
   tez_schema_evolution.q,\
   tez_union.q,\
-  tez_union_decimal.q
+  tez_union_decimal.q,\
+  tez_union_group_by.q,\
+  tez_smb_main.q,\
+  tez_smb_1.q,\
+  vectorized_dynamic_partition_pruning.q
 
 beeline.positive.exclude=add_part_exist.q,\
   alter1.q,\
@@ -342,6 +398,7 @@ beeline.positive.exclude=add_part_exist.
 
 minimr.query.negative.files=cluster_tasklog_retrieval.q,\
   file_with_header_footer_negative.q,\
+  local_mapred_error_cache.q,\
   mapreduce_stack_trace.q,\
   mapreduce_stack_trace_hadoop20.q,\
   mapreduce_stack_trace_turnoff.q,\

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Sun Oct  5 22:26:43 2014
@@ -690,7 +690,10 @@ public class QTestUtil {
     // conf.logVars(System.out);
     // System.out.flush();
 
+    String execEngine = conf.get("hive.execution.engine");
+    conf.set("hive.execution.engine", "mr");
     SessionState.start(conf);
+    conf.set("hive.execution.engine", execEngine);
     db = Hive.get(conf);
     fs = FileSystem.get(conf);
     drv = new Driver(conf);
@@ -771,6 +774,8 @@ public class QTestUtil {
     HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
         "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
 
+    String execEngine = conf.get("hive.execution.engine");
+    conf.set("hive.execution.engine", "mr");
     CliSessionState ss = new CliSessionState(conf);
     assert ss != null;
     ss.in = System.in;
@@ -788,6 +793,7 @@ public class QTestUtil {
 
     isSessionStateStarted = true;
 
+    conf.set("hive.execution.engine", execEngine);
     return ss;
   }
 

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDAFTestMax.java Sun Oct  5 22:26:43 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.udf;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDAF;
 import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
@@ -32,6 +33,8 @@ import org.apache.hadoop.io.Text;
  * UDAFTestMax.
  *
  */
+@Description(name = "test_max",
+value = "_FUNC_(col) - UDF to report Max Value")
 public class UDAFTestMax extends UDAF {
 
   /**

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFFileLookup.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFFileLookup.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFFileLookup.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFFileLookup.java Sun Oct  5 22:26:43 2014
@@ -27,6 +27,7 @@ import java.util.Map;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
@@ -34,6 +35,8 @@ import org.apache.hadoop.io.Text;
 /**
  * A UDF for testing, which does key/value lookup from a file
  */
+@Description(name = "lookup",
+value = "_FUNC_(col) - UDF for key/value lookup from a file")
 public class UDFFileLookup extends UDF {
   static Log LOG = LogFactory.getLog(UDFFileLookup.class);
 

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java Sun Oct  5 22:26:43 2014
@@ -18,11 +18,14 @@
 
 package org.apache.hadoop.hive.ql.udf;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 /**
  * A UDF for testing, which throws RuntimeException if  the length of a string.
  */
+@Description(name = "test_error",
+value = "_FUNC_(col) - UDF throws RuntimeException if  expression evaluates to false")
 public class UDFTestErrorOnFalse extends UDF {
 
   public int evaluate(Boolean b) {

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength.java Sun Oct  5 22:26:43 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.udf;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDF;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
@@ -25,6 +26,8 @@ import org.apache.hadoop.io.Text;
 /**
  * A UDF for testing, which evaluates the length of a string.
  */
+@Description(name = "testlength",
+value = "_FUNC_(col) - UDF evaluates the length of the string")
 public class UDFTestLength extends UDF {
 
   IntWritable result = new IntWritable();

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength2.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength2.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength2.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/UDFTestLength2.java Sun Oct  5 22:26:43 2014
@@ -18,12 +18,15 @@
 
 package org.apache.hadoop.hive.ql.udf;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDF;
 
 /**
  * A UDF for testing, which evaluates the length of a string. This UDF uses Java
  * Primitive classes for parameters.
  */
+@Description(name = "testlength2",
+value = "_FUNC_(col) - UDF evaluates the length of the string and returns value as Java Integer")
 public class UDFTestLength2 extends UDF {
 
   public Integer evaluate(String s) {

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/DummyContextUDF.java Sun Oct  5 22:26:43 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.udf.generic;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.MapredContext;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -26,7 +27,8 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.mapred.Counters;
 import org.apache.hadoop.mapred.Reporter;
-
+@Description(name = "counter",
+value = "_FUNC_(col) - UDF to report MR counter values")
 public class DummyContextUDF extends GenericUDF {
 
   private MapredContext context;

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaBoolean.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaBoolean.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaBoolean.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaBoolean.java Sun Oct  5 22:26:43 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.udf.generic;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -27,6 +28,8 @@ import org.apache.hadoop.hive.serde2.obj
 /**
  * A test GenericUDF to return native Java's boolean type
  */
+@Description(name = "test_udf_get_java_boolean",
+value = "_FUNC_(str) - GenericUDF to return native Java's boolean type")
 public class GenericUDFTestGetJavaBoolean extends GenericUDF {
   ObjectInspector[] argumentOIs;
 

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaString.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaString.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaString.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestGetJavaString.java Sun Oct  5 22:26:43 2014
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.udf.generic;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -27,6 +28,8 @@ import org.apache.hadoop.hive.serde2.obj
 /**
  * A test GenericUDF to return native Java's string type
  */
+@Description(name = "test_udf_get_java_string",
+value = "_FUNC_(str) - GenericUDF to return native Java's string type")
 public class GenericUDFTestGetJavaString extends GenericUDF {
   ObjectInspector[] argumentOIs;
 

Modified: hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java (original)
+++ hive/branches/spark-new/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTestTranslate.java Sun Oct  5 22:26:43 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.udf.ge
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
@@ -34,6 +35,8 @@ import org.apache.hadoop.io.Text;
 /**
  * Mimics oracle's function translate(str1, str2, str3).
  */
+@Description(name = "test_translate",
+value = "_FUNC_(str1, str2, str3) - Mimics oracle's function translate(str1, str2, str3)")
 public class GenericUDFTestTranslate extends GenericUDF {
   private transient ObjectInspector[] argumentOIs;
 

Modified: hive/branches/spark-new/jdbc/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/jdbc/pom.xml?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/jdbc/pom.xml (original)
+++ hive/branches/spark-new/jdbc/pom.xml Sun Oct  5 22:26:43 2014
@@ -41,8 +41,14 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
-      <artifactId>hive-metastore</artifactId>
+      <artifactId>hive-service</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+            <artifactId>hive-exec</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
@@ -51,12 +57,12 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
-      <artifactId>hive-service</artifactId>
+      <artifactId>hive-metastore</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
-      <artifactId>hive-exec</artifactId>
+      <artifactId>hive-shims</artifactId>
       <version>${project.version}</version>
     </dependency>
     <!-- inter-project -->
@@ -135,8 +141,39 @@
               <minimizeJar>true</minimizeJar>
               <shadedArtifactAttached>true</shadedArtifactAttached>
               <shadedClassifierName>${hive.jdbc.driver.classifier}</shadedClassifierName>
+              <filters>
+                <filter>
+                  <artifact>org.apache.hive.shims:hive-shims-common</artifact>
+                  <includes>
+                    <include>org/apache/hadoop/hive/shims/*</include>
+                    <include>org/apache/hadoop/hive/thrift/*</include>
+                  </includes>
+                </filter>
+                <filter>
+                  <artifact>org.apache.hive.shims:hive-shims-common-secure</artifact>
+                  <includes>
+                    <include>org/apache/hadoop/hive/thrift/*</include>
+                    <include>org/apache/hadoop/hive/thrift/client/*</include>
+                  </includes>
+                </filter>
+                <filter>
+                  <artifact>org.apache.hive.shims:hive-shims-0.23</artifact>
+                  <includes>
+                    <include>org/apache/hadoop/hive/thrift/*</include>
+                  </includes>
+                </filter>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
               <artifactSet>
                 <excludes>
+                  <exclude>org.apache.commons:commons-compress</exclude>
                   <exclude>org.apache.hadoop:*</exclude>
                   <exclude>org.apache.hive:hive-ant</exclude>
                   <exclude>org.apache.ant:*</exclude>
@@ -150,23 +187,16 @@
                   <exclude>org.tukaani:*</exclude>
                   <exclude>org.iq80.snappy:*</exclude>
                   <exclude>org.apache.velocity:*</exclude>
+                  <exclude>net.sf.jpam:*</exclude>
+                  <exclude>org.apache.avro:*</exclude>
+                  <exclude>net.sf.opencsv:*</exclude>
+                  <exclude>org.antlr:*</exclude>
                 </excludes>
-              </artifactSet>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/*.SF</exclude>
-                    <exclude>META-INF/*.DSA</exclude>
-                    <exclude>META-INF/*.RSA</exclude>
-                  </excludes>
-                </filter>
-              </filters>
+              </artifactSet>       
             </configuration>
           </execution>
         </executions>
       </plugin>
     </plugins>
   </build>
-
 </project>

Modified: hive/branches/spark-new/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDriver.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDriver.java (original)
+++ hive/branches/spark-new/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveDriver.java Sun Oct  5 22:26:43 2014
@@ -102,8 +102,9 @@ public class HiveDriver implements Drive
     return Pattern.matches(URL_PREFIX + ".*", url);
   }
 
+  @Override
   public Connection connect(String url, Properties info) throws SQLException {
-    return new HiveConnection(url, info);
+    return acceptsURL(url) ? new HiveConnection(url, info) : null;
   }
 
   /**

Modified: hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java (original)
+++ hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveBaseResultSet.java Sun Oct  5 22:26:43 2014
@@ -58,6 +58,7 @@ public abstract class HiveBaseResultSet 
   protected boolean wasNull = false;
   protected Object[] row;
   protected List<String> columnNames;
+  protected List<String> normalizedColumnNames;
   protected List<String> columnTypes;
   protected List<JdbcColumnAttributes> columnAttributes;
 
@@ -84,7 +85,7 @@ public abstract class HiveBaseResultSet 
   }
 
   public int findColumn(String columnName) throws SQLException {
-    int columnIndex = columnNames.indexOf(columnName);
+    int columnIndex = normalizedColumnNames.indexOf(columnName.toLowerCase());
     if (columnIndex==-1) {
       throw new SQLException();
     } else {

Modified: hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveMetaDataResultSet.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveMetaDataResultSet.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveMetaDataResultSet.java (original)
+++ hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveMetaDataResultSet.java Sun Oct  5 22:26:43 2014
@@ -36,8 +36,13 @@ public abstract class HiveMetaDataResult
     }
     if (columnNames!=null) {
       this.columnNames = new ArrayList<String>(columnNames);
+      this.normalizedColumnNames = new ArrayList<String>();
+      for (String colName : columnNames) {
+        this.normalizedColumnNames.add(colName.toLowerCase());
+      }
     } else {
       this.columnNames =  new ArrayList<String>();
+      this.normalizedColumnNames = new ArrayList<String>();
     }
     if (columnTypes!=null) {
       this.columnTypes = new ArrayList<String>(columnTypes);

Modified: hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java (original)
+++ hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java Sun Oct  5 22:26:43 2014
@@ -28,6 +28,7 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -75,6 +76,7 @@ public class HiveQueryResultSet extends 
   private boolean fetchFirst = false;
 
   private final TProtocolVersion protocol;
+  private ReentrantLock transportLock;
 
 
   public static class Builder {
@@ -98,6 +100,7 @@ public class HiveQueryResultSet extends 
     private int fetchSize = 50;
     private boolean emptyResultSet = false;
     private boolean isScrollable = false;
+    private ReentrantLock transportLock = null;
 
     public Builder(Statement statement) throws SQLException {
       this.statement = statement;
@@ -166,6 +169,11 @@ public class HiveQueryResultSet extends 
       return this;
     }
 
+    public Builder setTransportLock(ReentrantLock transportLock) {
+      this.transportLock = transportLock;
+      return this;
+    }
+
     public HiveQueryResultSet build() throws SQLException {
       return new HiveQueryResultSet(this);
     }
@@ -181,7 +189,9 @@ public class HiveQueryResultSet extends 
     this.stmtHandle = builder.stmtHandle;
     this.sessHandle = builder.sessHandle;
     this.fetchSize = builder.fetchSize;
+    this.transportLock = builder.transportLock;
     columnNames = new ArrayList<String>();
+    normalizedColumnNames = new ArrayList<String>();
     columnTypes = new ArrayList<String>();
     columnAttributes = new ArrayList<JdbcColumnAttributes>();
     if (builder.retrieveSchema) {
@@ -239,7 +249,17 @@ public class HiveQueryResultSet extends 
     try {
       TGetResultSetMetadataReq metadataReq = new TGetResultSetMetadataReq(stmtHandle);
       // TODO need session handle
-      TGetResultSetMetadataResp  metadataResp = client.GetResultSetMetadata(metadataReq);
+      TGetResultSetMetadataResp  metadataResp;
+      if (transportLock == null) {
+        metadataResp = client.GetResultSetMetadata(metadataReq);
+      } else {
+        transportLock.lock();
+        try {
+          metadataResp = client.GetResultSetMetadata(metadataReq);
+        } finally {
+          transportLock.unlock();
+        }
+      }
       Utils.verifySuccess(metadataResp.getStatus());
 
       StringBuilder namesSb = new StringBuilder();
@@ -260,6 +280,7 @@ public class HiveQueryResultSet extends 
         }
         String columnName = columns.get(pos).getColumnName();
         columnNames.add(columnName);
+        normalizedColumnNames.add(columnName.toLowerCase());
         TPrimitiveTypeEntry primitiveTypeEntry =
             columns.get(pos).getTypeDesc().getTypes().get(0).getPrimitiveEntry();
         String columnTypeName = TYPE_NAMES.get(primitiveTypeEntry.getType());
@@ -284,6 +305,10 @@ public class HiveQueryResultSet extends 
     columnNames.addAll(colNames);
     columnTypes.addAll(colTypes);
     columnAttributes.addAll(colAttributes);
+
+    for (String colName : colNames) {
+      normalizedColumnNames.add(colName.toLowerCase());
+    }
   }
 
   @Override
@@ -326,7 +351,17 @@ public class HiveQueryResultSet extends 
       if (fetchedRows == null || !fetchedRowsItr.hasNext()) {
         TFetchResultsReq fetchReq = new TFetchResultsReq(stmtHandle,
             orientation, fetchSize);
-        TFetchResultsResp fetchResp = client.FetchResults(fetchReq);
+        TFetchResultsResp fetchResp;
+        if (transportLock == null) {
+          fetchResp = client.FetchResults(fetchReq);
+        } else {
+          transportLock.lock();
+          try {
+            fetchResp = client.FetchResults(fetchReq);
+          } finally {
+            transportLock.unlock();
+          }
+        }
         Utils.verifySuccessWithInfo(fetchResp.getStatus());
 
         TRowSet results = fetchResp.getResults();

Modified: hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java (original)
+++ hive/branches/spark-new/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java Sun Oct  5 22:26:43 2014
@@ -23,10 +23,14 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
 import java.sql.SQLWarning;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.locks.ReentrantLock;
 
+import org.apache.hive.service.cli.RowSet;
+import org.apache.hive.service.cli.RowSetFactory;
 import org.apache.hive.service.cli.thrift.TCLIService;
 import org.apache.hive.service.cli.thrift.TCancelOperationReq;
 import org.apache.hive.service.cli.thrift.TCancelOperationResp;
@@ -38,6 +42,9 @@ import org.apache.hive.service.cli.thrif
 import org.apache.hive.service.cli.thrift.TGetOperationStatusResp;
 import org.apache.hive.service.cli.thrift.TOperationHandle;
 import org.apache.hive.service.cli.thrift.TSessionHandle;
+import org.apache.hive.service.cli.thrift.TFetchResultsReq;
+import org.apache.hive.service.cli.thrift.TFetchResultsResp;
+import org.apache.hive.service.cli.thrift.TFetchOrientation;
 
 /**
  * HiveStatement.
@@ -77,6 +84,27 @@ public class HiveStatement implements ja
    */
   private boolean isClosed = false;
 
+  /**
+   * Keep state so we can fail certain calls made after cancel().
+   */
+  private boolean isCancelled = false;
+
+  /**
+   * Keep this state so we can know whether the query in this statement is closed.
+   */
+  private boolean isQueryClosed = false;
+
+  /**
+   * Keep this state so we can know whether the query logs are being generated in HS2.
+   */
+  private boolean isLogBeingGenerated = true;
+
+  /**
+   * Keep this state so we can know whether the statement is submitted to HS2 and start execution
+   * successfully.
+   */
+  private boolean isExecuteStatementFailed = false;
+
   // A fair reentrant lock
   private ReentrantLock transportLock = new ReentrantLock(true);
 
@@ -113,6 +141,9 @@ public class HiveStatement implements ja
   @Override
   public void cancel() throws SQLException {
     checkConnection("cancel");
+    if (isCancelled) {
+      return;
+    }
 
     transportLock.lock();
     try {
@@ -128,6 +159,7 @@ public class HiveStatement implements ja
     } finally {
       transportLock.unlock();
     }
+    isCancelled = true;
   }
 
   /*
@@ -167,6 +199,8 @@ public class HiveStatement implements ja
     } finally {
       transportLock.unlock();
     }
+    isQueryClosed = true;
+    isExecuteStatementFailed = false;
     stmtHandle = null;
   }
 
@@ -202,6 +236,7 @@ public class HiveStatement implements ja
     checkConnection("execute");
 
     closeClientOperation();
+    initFlags();
 
     TExecuteStatementReq execReq = new TExecuteStatementReq(sessHandle, sql);
     /**
@@ -218,9 +253,12 @@ public class HiveStatement implements ja
       TExecuteStatementResp execResp = client.ExecuteStatement(execReq);
       Utils.verifySuccessWithInfo(execResp.getStatus());
       stmtHandle = execResp.getOperationHandle();
+      isExecuteStatementFailed = false;
     } catch (SQLException eS) {
+      isExecuteStatementFailed = true;
       throw eS;
     } catch (Exception ex) {
+      isExecuteStatementFailed = true;
       throw new SQLException(ex.toString(), "08S01", ex);
     } finally {
       transportLock.unlock();
@@ -266,11 +304,14 @@ public class HiveStatement implements ja
           }
         }
       } catch (SQLException e) {
+        isLogBeingGenerated = false;
         throw e;
       } catch (Exception e) {
+        isLogBeingGenerated = false;
         throw new SQLException(e.toString(), "08S01", e);
       }
     }
+    isLogBeingGenerated = false;
 
     // The query should be completed by now
     if (!stmtHandle.isHasResultSet()) {
@@ -278,7 +319,7 @@ public class HiveStatement implements ja
     }
     resultSet =  new HiveQueryResultSet.Builder(this).setClient(client).setSessionHandle(sessHandle)
         .setStmtHandle(stmtHandle).setMaxRows(maxRows).setFetchSize(fetchSize)
-        .setScrollable(isScrollableResultset)
+        .setScrollable(isScrollableResultset).setTransportLock(transportLock)
         .build();
     return true;
   }
@@ -289,6 +330,13 @@ public class HiveStatement implements ja
     }
   }
 
+  private void initFlags() {
+    isCancelled = false;
+    isQueryClosed = false;
+    isLogBeingGenerated = true;
+    isExecuteStatementFailed = false;
+  }
+
   /*
    * (non-Javadoc)
    *
@@ -713,4 +761,93 @@ public class HiveStatement implements ja
     throw new SQLException("Cannot unwrap to " + iface);
   }
 
+  /**
+   * Check whether query execution might be producing more logs to be fetched.
+   * This method is a public API for usage outside of Hive, although it is not part of the
+   * interface java.sql.Statement.
+   * @return true if query execution might be producing more logs. It does not indicate if last
+   *         log lines have been fetched by getQueryLog.
+   */
+  public boolean hasMoreLogs() {
+    return isLogBeingGenerated;
+  }
+
+  /**
+   * Get the execution logs of the given SQL statement.
+   * This method is a public API for usage outside of Hive, although it is not part of the
+   * interface java.sql.Statement.
+   * This method gets the incremental logs during SQL execution, and uses fetchSize holden by
+   * HiveStatement object.
+   * @return a list of logs. It can be empty if there are no new logs to be retrieved at that time.
+   * @throws SQLException
+   * @throws ClosedOrCancelledStatementException if statement has been cancelled or closed
+   */
+  public List<String> getQueryLog() throws SQLException, ClosedOrCancelledStatementException {
+    return getQueryLog(true, fetchSize);
+  }
+
+  /**
+   * Get the execution logs of the given SQL statement.
+   * This method is a public API for usage outside of Hive, although it is not part of the
+   * interface java.sql.Statement.
+   * @param incremental indicate getting logs either incrementally or from the beginning,
+   *                    when it is true or false.
+   * @param fetchSize the number of lines to fetch
+   * @return a list of logs. It can be empty if there are no new logs to be retrieved at that time.
+   * @throws SQLException
+   * @throws ClosedOrCancelledStatementException if statement has been cancelled or closed
+   */
+  public List<String> getQueryLog(boolean incremental, int fetchSize)
+      throws SQLException, ClosedOrCancelledStatementException {
+    checkConnection("getQueryLog");
+    if (isCancelled) {
+      throw new ClosedOrCancelledStatementException("Method getQueryLog() failed. The " +
+          "statement has been closed or cancelled.");
+    }
+
+    List<String> logs = new ArrayList<String>();
+    TFetchResultsResp tFetchResultsResp = null;
+    transportLock.lock();
+    try {
+      if (stmtHandle != null) {
+        TFetchResultsReq tFetchResultsReq = new TFetchResultsReq(stmtHandle,
+            getFetchOrientation(incremental), fetchSize);
+        tFetchResultsReq.setFetchType((short)1);
+        tFetchResultsResp = client.FetchResults(tFetchResultsReq);
+        Utils.verifySuccessWithInfo(tFetchResultsResp.getStatus());
+      } else {
+        if (isQueryClosed) {
+          throw new ClosedOrCancelledStatementException("Method getQueryLog() failed. The " +
+              "statement has been closed or cancelled.");
+        }
+        if (isExecuteStatementFailed) {
+          throw new SQLException("Method getQueryLog() failed. Because the stmtHandle in " +
+              "HiveStatement is null and the statement execution might fail.");
+        } else {
+          return logs;
+        }
+      }
+    } catch (SQLException e) {
+      throw e;
+    } catch (Exception e) {
+      throw new SQLException("Error when getting query log: " + e, e);
+    } finally {
+      transportLock.unlock();
+    }
+
+    RowSet rowSet = RowSetFactory.create(tFetchResultsResp.getResults(),
+        connection.getProtocol());
+    for (Object[] row : rowSet) {
+      logs.add((String)row[0]);
+    }
+    return logs;
+  }
+
+  private TFetchOrientation getFetchOrientation(boolean incremental) {
+    if (incremental) {
+      return TFetchOrientation.FETCH_NEXT;
+    } else {
+      return TFetchOrientation.FETCH_FIRST;
+    }
+  }
 }

Modified: hive/branches/spark-new/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql (original)
+++ hive/branches/spark-new/metastore/scripts/upgrade/mssql/hive-schema-0.14.0.mssql.sql Sun Oct  5 22:26:43 2014
@@ -836,14 +836,14 @@ CREATE INDEX TABLE_PARAMS_N49 ON TABLE_P
 -- These are not part of package jdo, so if you are going to regenerate this file you need to manually add the following section back to the file.
 -- -----------------------------------------------------------------------------------------------------------------------------------------------
 CREATE TABLE COMPACTION_QUEUE(
-	CQ_ID int NOT NULL,
+	CQ_ID bigint NOT NULL,
 	CQ_DATABASE varchar(128) NOT NULL,
 	CQ_TABLE varchar(128) NOT NULL,
 	CQ_PARTITION varchar(767) NULL,
 	CQ_STATE char(1) NOT NULL,
 	CQ_TYPE char(1) NOT NULL,
 	CQ_WORKER_ID varchar(128) NULL,
-	CQ_START int NULL,
+	CQ_START bigint NULL,
 	CQ_RUN_AS varchar(128) NULL,
 PRIMARY KEY CLUSTERED 
 (
@@ -852,23 +852,23 @@ PRIMARY KEY CLUSTERED 
 );
 
 CREATE TABLE COMPLETED_TXN_COMPONENTS(
-	CTC_TXNID int NULL,
+	CTC_TXNID bigint NULL,
 	CTC_DATABASE varchar(128) NOT NULL,
 	CTC_TABLE varchar(128) NULL,
 	CTC_PARTITION varchar(767) NULL
 );
 
 CREATE TABLE HIVE_LOCKS(
-	HL_LOCK_EXT_ID int NOT NULL,
-	HL_LOCK_INT_ID int NOT NULL,
-	HL_TXNID int NULL,
+	HL_LOCK_EXT_ID bigint NOT NULL,
+	HL_LOCK_INT_ID bigint NOT NULL,
+	HL_TXNID bigint NULL,
 	HL_DB varchar(128) NOT NULL,
 	HL_TABLE varchar(128) NULL,
 	HL_PARTITION varchar(767) NULL,
 	HL_LOCK_STATE char(1) NOT NULL,
 	HL_LOCK_TYPE char(1) NOT NULL,
-	HL_LAST_HEARTBEAT int NOT NULL,
-	HL_ACQUIRED_AT int NULL,
+	HL_LAST_HEARTBEAT bigint NOT NULL,
+	HL_ACQUIRED_AT bigint NULL,
 	HL_USER varchar(128) NOT NULL,
 	HL_HOST varchar(128) NOT NULL,
 PRIMARY KEY CLUSTERED 
@@ -879,28 +879,28 @@ PRIMARY KEY CLUSTERED 
 );
 
 CREATE TABLE NEXT_COMPACTION_QUEUE_ID(
-	NCQ_NEXT int NOT NULL
+	NCQ_NEXT bigint NOT NULL
 );
 
 INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1);
 
 CREATE TABLE NEXT_LOCK_ID(
-	NL_NEXT int NOT NULL
+	NL_NEXT bigint NOT NULL
 );
 
 INSERT INTO NEXT_LOCK_ID VALUES(1);
 
 CREATE TABLE NEXT_TXN_ID(
-	NTXN_NEXT int NOT NULL
+	NTXN_NEXT bigint NOT NULL
 );
 
 INSERT INTO NEXT_TXN_ID VALUES(1);
 
 CREATE TABLE TXNS(
-	TXN_ID int NOT NULL,
+	TXN_ID bigint NOT NULL,
 	TXN_STATE char(1) NOT NULL,
-	TXN_STARTED int NOT NULL,
-	TXN_LAST_HEARTBEAT int NOT NULL,
+	TXN_STARTED bigint NOT NULL,
+	TXN_LAST_HEARTBEAT bigint NOT NULL,
 	TXN_USER varchar(128) NOT NULL,
 	TXN_HOST varchar(128) NOT NULL,
 PRIMARY KEY CLUSTERED 
@@ -910,7 +910,7 @@ PRIMARY KEY CLUSTERED 
 );
 
 CREATE TABLE TXN_COMPONENTS(
-	TC_TXNID int NULL,
+	TC_TXNID bigint NULL,
 	TC_DATABASE varchar(128) NOT NULL,
 	TC_TABLE varchar(128) NULL,
 	TC_PARTITION varchar(767) NULL

Modified: hive/branches/spark-new/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql (original)
+++ hive/branches/spark-new/metastore/scripts/upgrade/mssql/upgrade-0.13.0-to-0.14.0.mssql.sql Sun Oct  5 22:26:43 2014
@@ -1,6 +1,7 @@
 SELECT 'Upgrading MetaStore schema from 0.13.0 to 0.14.0' AS MESSAGE;
 
 :r 002-HIVE-7784.mssql.sql;
+:r 003-HIVE-8239.mssql.sql;
 
 UPDATE VERSION SET SCHEMA_VERSION='0.14.0', VERSION_COMMENT='Hive release version 0.14.0' where VER_ID=1;
 SELECT 'Finished upgrading MetaStore schema from 0.13.0 to 0.14.0' AS MESSAGE;

Modified: hive/branches/spark-new/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql
URL: http://svn.apache.org/viewvc/hive/branches/spark-new/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql?rev=1629544&r1=1629543&r2=1629544&view=diff
==============================================================================
--- hive/branches/spark-new/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql (original)
+++ hive/branches/spark-new/metastore/scripts/upgrade/oracle/upgrade-0.13.0-to-0.14.0.oracle.sql Sun Oct  5 22:26:43 2014
@@ -1,5 +1,6 @@
 SELECT 'Upgrading MetaStore schema from 0.13.0 to 0.14.0' AS Status from dual;
 
+@019-HIVE-7118.oracle.sql;
 @020-HIVE-7784.oracle.sql;
 
 UPDATE VERSION SET SCHEMA_VERSION='0.14.0', VERSION_COMMENT='Hive release version 0.14.0' where VER_ID=1;



Mime
View raw message